# Robots.txt for SuccessHabits.com
# This file tells search engines how to crawl and index the website

# Allow all search engines
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /assets/test/

# Crawl delay (optional, in seconds)
Crawl-delay: 1

# Request rate (optional, in requests per second)
Request-rate: 10

# Specific rules for Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Specific rules for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block bad bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Sitemap location
Sitemap: https://successhabits.com/sitemap.xml

# Specify canonical homepage
# This helps consolidate search engine rankings