# FCA Reports Australia — robots.txt
# https://fcareportsaustralia.com.au/

User-agent: *
Allow: /

# Block any admin/internal/preview-only paths
Disallow: /admin
Disallow: /api/

# Allow major search crawlers explicitly
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

# Block known scrapers / aggressive bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# Sitemap
Sitemap: https://fcareportsaustralia.com.au/sitemap.xml
Sitemap: https://fcareportsaustralia.com.au/api/sitemap-articles.xml

# Host directive (helps with canonical host signal)
Host: https://fcareportsaustralia.com.au