# robots.txt — mitbbs.ai (华人空间)
# 2026-10-16 Worker F: GEO audit P1-12 tiered bot policy (owner-approved 2026-10-16).
# This replaces the 2026-10-10 GPTBot-only group. (The "~40 AI crawlers" blanket
# block noted in memory never materialized in this file — Bot Fight Mode at
# the Cloudflare edge remains the other layer; verify its AI-bot toggle there.)
#
# Three bot classes:
#   1. "*" (default): full site minus admin / AJAX / state URLs.
#   2. Search/retrieval bots: allowed EVERYWHERE (earn citations + traffic).
#      Includes Meta-ExternalAgent + Meta-ExternalFetcher per the 2026-10-16
#      owner approval (supersedes the 2026-10-04 "keep meta-externalagent
#      blocked" note).
#   3. Training crawlers (GPTBot / ClaudeBot / Google-Extended / CCBot):
#      FIRST-PARTY content ONLY. robots.txt is path-based and cannot split
#      per-topic rights — aggregated_third_party full text lives in 30+ forums
#      (see phpbb_mitbbs_topic_rights) — so training bots are kept off raw
#      topic HTML (/t/, /viewtopic.php and the /zh-hant/, /en/ variants, plus
#      the curated-aggregation topic hubs /app.php/topic/).
# INTERIM 2026-10-17 (W-R1, review R1 P0): /app.php/md/ is ALSO disallowed for
# training crawlers until the per-topic rights gate on the twins is verified
# fixed (it previously served full aggregated text). Re-point training bots at
# the twins only AFTER the gate fix is confirmed live.
#      Allowed first-party surfaces: landing, data pages (finance/sports/
#      weather/currency/yellowpages/visa/fantasy), agent docs, llms.txt,
#      blogs, forum listings (titles/excerpts).
#
# NOTE: a bot matching a named group ignores the "*" group, so the shared
# disallows are repeated verbatim inside each named group.

User-agent: *
Allow: /
Disallow: /adm/
Disallow: /app.php/pages/admin
Disallow: /app.php/email/admin
Disallow: /mail/
Disallow: /ucp.php
Disallow: /mcp.php
Disallow: /search.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /app.php/agent-api/
Disallow: /app.php/likes/
Disallow: /pinned_api.php
Disallow: /app.php/pages/track
Disallow: /i18n.php
Disallow: /*?*sid=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*mark=
Disallow: /*?*hilit=
Disallow: /*?*watch=
Disallow: /images/news/  # 2026-10-17 W-N1 P0: archived third-party images must not be crawled

Sitemap: https://mitbbs.ai/sitemap.xml
Sitemap: https://mitbbs.ai/sitemap-news.xml

# ---- Class 1b: social card crawlers — default rules, news images allowed ----
# 2026-10-30 fix: Twitterbot (X) and facebookexternalhit (Meta/Threads) had
# no named group, so the "*" group's W-N1 disallow of /images/news/ applied
# to them, and every share card whose image is a re-hosted news photo
# rendered as an empty grey placeholder (the page itself was allowed, so
# title/description came through — only the image fetch was robots-blocked).
# These two crawlers fetch an image only to render a card for a link a user
# shared; they do not index images. They therefore get the default group's
# rules WITHOUT the /images/news/ line. The disallow stands for all others.
User-agent: Twitterbot
User-agent: facebookexternalhit
Allow: /
Disallow: /adm/
Disallow: /app.php/pages/admin
Disallow: /app.php/email/admin
Disallow: /mail/
Disallow: /ucp.php
Disallow: /mcp.php
Disallow: /search.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /app.php/agent-api/
Disallow: /app.php/likes/
Disallow: /pinned_api.php
Disallow: /app.php/pages/track
Disallow: /i18n.php
Disallow: /*?*sid=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*mark=
Disallow: /*?*hilit=
Disallow: /*?*watch=

# ---- Class 2: search/retrieval bots — allowed everywhere (owner-approved) ----
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /adm/
Disallow: /app.php/pages/admin
Disallow: /app.php/email/admin
Disallow: /mail/
Disallow: /ucp.php
Disallow: /mcp.php
Disallow: /search.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /app.php/agent-api/
Disallow: /app.php/likes/
Disallow: /pinned_api.php
Disallow: /app.php/pages/track
Disallow: /i18n.php
Disallow: /*?*sid=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*mark=
Disallow: /*?*hilit=
Disallow: /*?*watch=
Disallow: /images/news/  # 2026-10-17 W-N1 P0: archived third-party images must not be crawled

# ---- Class 3: training crawlers — first-party content ONLY (owner-approved) ----
# Raw topic HTML is off-limits (per-topic rights not expressible here).
# INTERIM 2026-10-17 (W-R1): /app.php/md/ disallowed for training crawlers too
# (P0 leak: twins served full aggregated text; gate fix must verify first).
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /agents.php
Allow: /agents.md
Allow: /finance.php
Allow: /sports.php
Allow: /weather.php
Allow: /currency.php
Allow: /yellowpages.php
Allow: /visa.php
Allow: /fantasy.php
Allow: /app.php/blog
Disallow: /app.php/md/
Disallow: /t/
Disallow: /viewtopic.php
Disallow: /zh-hant/
Disallow: /en/
Disallow: /i18n.php
Disallow: /app.php/topic/
Disallow: /adm/
Disallow: /app.php/pages/admin
Disallow: /app.php/email/admin
Disallow: /mail/
Disallow: /ucp.php
Disallow: /mcp.php
Disallow: /search.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /app.php/agent-api/
Disallow: /app.php/likes/
Disallow: /pinned_api.php
Disallow: /app.php/pages/track
Disallow: /*?*sid=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*mark=
Disallow: /*?*hilit=
Disallow: /*?*watch=
Disallow: /images/news/  # 2026-10-17 W-N1 P0: archived third-party images must not be crawled

# 2026-10-09: Block Bytespider (low citation value, does not always honor robots.txt)
User-agent: Bytespider
Disallow: /

# ---- Exa crawlers: FULLY BLOCKED (owner order 2026-10-30) ----
# Supersedes the 2026-10-17 rate-limit compromise: ExaSearchBot/Exabot reached
# 27k hits/day with 62.6% of crawl landing on thin/noindex pages (crawl-watch
# 2026-10-30). Hard block also enforced at the server via .htaccess (403 by UA).
User-agent: ExaSearchBot
User-agent: Exabot
User-agent: ExaUser
Disallow: /

# ---- AhrefsBot: FULLY BLOCKED (owner order 2026-10-30) ----
# Second-largest named crawler (5,640 hits/24h, 8.5% of bot traffic,
# crawl-watch 2026-10-30); SEO backlink index - no readers, no citations.
# Hard block also enforced at the server via .htaccess (403 by UA).
User-agent: AhrefsBot
Disallow: /