# ============================================
# 法律偵探 dr-legal.com.tw — robots.txt
# ============================================
# 策略 (2026-05-12 SEO/GEO Phase 3, codex 拍板):
#   - 允許 search/citation bots 索引公開判決頁與首頁
#   - 暫不開 training bots (GPTBot/ClaudeBot/Google-Extended)
#     這三個是訓練授權策略, 不是搜尋曝光必要條件
#   - 後台/API/個人資料路徑全擋
# ============================================

# --- 預設規則 (所有未明確列出的 bot 套用此規則) ---
User-agent: *
Allow: /
Allow: /fullview/
Allow: /square/
Allow: /mcp
Allow: /topics
Allow: /reports
Allow: /legal-ai-tools
Allow: /enterprise
Allow: /about
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- Google 主要搜尋爬蟲 ---
User-agent: Googlebot
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- Bing / Copilot ---
User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- OpenAI 搜尋與引用 (非訓練) ---
# OAI-SearchBot 是 ChatGPT search 即時引用爬蟲, 跟 GPTBot 訓練爬蟲分開
User-agent: OAI-SearchBot
Allow: /fullview/
Allow: /square/
Allow: /mcp
Allow: /topics
Allow: /reports
Allow: /legal-ai-tools
Allow: /enterprise
Allow: /about
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- Anthropic Claude 搜尋與引用 (非訓練) ---
# Claude-SearchBot 是 Claude search 即時引用爬蟲, 跟 anthropic-ai 訓練爬蟲分開
User-agent: Claude-SearchBot
Allow: /fullview/
Allow: /square/
Allow: /mcp
Allow: /topics
Allow: /reports
Allow: /legal-ai-tools
Allow: /enterprise
Allow: /about
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- Perplexity 引用爬蟲 ---
User-agent: PerplexityBot
Allow: /fullview/
Allow: /square/
Allow: /mcp
Allow: /topics
Allow: /reports
Allow: /legal-ai-tools
Allow: /enterprise
Allow: /about
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- Training bots: 全站開放, 僅擋後台/API/個資路徑 (2026-08-22 Aaron 拍板) ---
# 判決全文=司法院公開資料; Square 僅出 anon_public; HTML 零機制洩漏(實測);
# robots 擋不住蒸餾與倒資料(真防線=slowapi+Cloudflare), 只擋官方訓練爬蟲,
# 而那正是「讓 AI 認識本站」要的。CCBot(Common Crawl)供開源模型訓練, 一併開放。
User-agent: GPTBot
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

User-agent: ClaudeBot
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

User-agent: anthropic-ai
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

User-agent: Google-Extended
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

User-agent: CCBot
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /profile
Disallow: /batch-analysis
Disallow: /login
Disallow: /account
Disallow: /ask/session/
Disallow: /_next/data/

# --- 封鎖純訓練用途爬蟲 / 不友善爬蟲 ---
User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# --- Sitemap ---
Sitemap: https://dr-legal.com.tw/sitemap.xml