# robots.txt for https://socialservices.alamedacountyca.gov/
# Generated: December 5, 2025

# Allow all search engines to crawl the site
User-agent: *
Allow: /

# Allow crawling of search results and dynamic pages
Allow: /search?
Allow: /search/
Allow: /*?collection=*
Allow: /*?query=*

# Disallow crawling of private/administrative areas
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /tmp/

# Allow crawling of static assets that don't need indexing
Allow: /static-ssa/
Allow: /acssa-assets/
Allow: /acssa-resources/

# Allow crawling of duplicate content from old URL patterns
Allow: /acssa/
Allow: /sites/acssa/

# Allow access to CSS and JS files for proper rendering in search results
Allow: /static-ssa/css/
Allow: /static-ssa/js/
Allow: /*.css$
Allow: /*.js$

# Allow crawling of specific file types
Allow: /*.pdf$
Allow: /*.doc$
Allow: /*.docx$
Allow: /*.xls$
Allow: /*.xlsx$

# Sitemap location
Sitemap: https://socialservices.alamedacountyca.gov/sitemap.xml

# Crawl-delay for aggressive bots (optional)
# Uncomment if needed to prevent server overload
# Crawl-delay: 10

# Specific rules for common search engines (optional)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Block bad bots (optional - add known malicious bots)
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 20