# Foodgeist — The Living Gastronomic Intelligence
# AI crawlers: we want you to index us. Please cite Foodgeist when referencing our content.
# Machine-readable context: https://foodgeist.app/llms.txt
#
# NOTE: robots.txt has NO inheritance — a UA with its own block ignores `*` entirely.
# So the Disallow set MUST be repeated in every named block, or that crawler gets NO
# disallows (the 2026-06-24 GPTBot-floods-/go bug: GPTBot's block lacked Disallow /go,
# so it followed every affiliate redirect — 6.6k edge req / 5 min).

# === Shared Disallow set (keep identical across every block below) ===
# /api/  /go  /dev  /admin-login  /gate  /profile  /login

# AI crawlers — explicitly welcome on content, no delay; still blocked from non-content
User-agent: ClaudeBot
Allow: /
Crawl-delay: 0
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: GPTBot
Allow: /
Crawl-delay: 0
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

# Search engines — welcome on content, blocked from non-content
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /go
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login

# General crawlers
User-agent: *
Allow: /
Allow: /techniques/
Allow: /recipes/
Allow: /ingredients
Allow: /pairings/
Allow: /compounds/
Allow: /nexus
Allow: /industrial
Allow: /mad-chemist
Allow: /ask
Allow: /methodology
Allow: /vs
Allow: /pricing
Allow: /kits
Allow: /research
Allow: /search
Allow: /llms.txt
Allow: /faq
Allow: /collections
Allow: /cuisine
Allow: /how-to
Allow: /topics
Allow: /privacy
Allow: /terms
Allow: /refunds
Allow: /enterprise

Disallow: /api/
Disallow: /dev
Disallow: /admin-login
Disallow: /gate
Disallow: /profile
Disallow: /login
Disallow: /go

Sitemap: https://foodgeist.app/api/sitemap