# Fuel Finder UK - robots.txt
# https://www.fuel-finder.uk/robots.txt

# ===========================================
# STANDARD WEB CRAWLERS
# ===========================================

User-agent: *
Allow: /
Allow: /api/csrf/$
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=
Crawl-delay: 1

# ===========================================
# GOOGLE
# ===========================================

User-agent: Googlebot
Allow: /
Allow: /api/csrf/$
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

User-agent: Googlebot-Image
Allow: /
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# ===========================================
# BING
# ===========================================

User-agent: Bingbot
Allow: /
Allow: /api/csrf/$
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# ===========================================
# AI SEARCH ASSISTANTS
# ===========================================

# OpenAI's ChatGPT web browsing
User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# OpenAI's GPTBot (for training)
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Google AI / Bard / Gemini
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Anthropic's Claude
User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Meta AI
User-agent: meta-externalagent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# Common Crawl (used by many AI companies)
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /geocode/
Disallow: /*?lat=
Disallow: /*?*&lat=

# ===========================================
# SOCIAL MEDIA CRAWLERS
# ===========================================

User-agent: Twitterbot
Allow: /
Disallow: /api/
Disallow: /geocode/

User-agent: LinkedInBot
Allow: /
Disallow: /api/
Disallow: /geocode/

User-agent: facebookexternalhit
Allow: /
Disallow: /api/
Disallow: /geocode/

# ===========================================
# SITEMAPS & LLM FILES
# ===========================================

Sitemap: https://www.fuel-finder.uk/sitemap.xml
Sitemap: https://www.fuel-finder.uk/core.xml
Sitemap: https://www.fuel-finder.uk/national.xml
Sitemap: https://www.fuel-finder.uk/location.xml
Sitemap: https://www.fuel-finder.uk/stations.xml
Sitemap: https://www.fuel-finder.uk/methodology.xml
Sitemap: https://www.fuel-finder.uk/blog/sitemap.xml

# AI-specific information files
# llms.txt - Summary for AI assistants
# llms-full.txt - Comprehensive reference