# Allow all web crawlers
User-agent: *

# Allow all content
Allow: /

# Explicitly allow news sections
Allow: /news
Allow: /news/
Allow: /news/*
Allow: /articles/*

# Common directories to disallow
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/

# Sitemap location (update with your domain)
Sitemap: https://yourdomain.com/sitemap.xml

# Crawl-delay for all bots (in seconds)
Crawl-delay: 5