User-agent: *
Allow: /
Disallow: /tags/
Disallow: /categories/
Disallow: /search/
Disallow: /api/
Disallow: /admin/
Disallow: /.git/
Disallow: /hugo.toml
Disallow: /README.md
Disallow: /go.mod
Disallow: /go.sum
Disallow: /.gitignore

# 特别允许重要的内容目录
Allow: /profile/
Allow: /posts/
Allow: /pictures/
Allow: /images/

# AI Search & retrieval crawlers: allow
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# AI User-triggered fetchers: allow
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# AI Training crawlers: allow（实体传播策略——让 ByF (BaiFan) 进入模型训练语料）
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Undeclared crawlers: block（未申报爬虫，对实体认知无贡献）
User-agent: Bytespider
Disallow: /

# 搜索引擎特定配置
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Sitemap位置 (robots.txt 只生成于根目录, 指向默认语言 sitemap)
Sitemap: https://blog.baifan.site/sitemap.xml

# llms.txt location for AI discovery
# https://blog.baifan.site/llms.txt