# StockExpertAI - robots.txt

# Allow Twitterbot full access (needed for Twitter/X card previews including ?tab= URLs)
User-agent: Twitterbot
Allow: /

# Social link-preview bots — full access so og:image fetches keep working
# after the global "Disallow: /api/" below (these groups override *).
User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

# AI retrieval / search / citation bots — ALLOW (real user-facing AI
# search traffic, generates citations). Each of these generates ~1
# referral per ~100 crawls, which is worth the bandwidth.
# RFC 9309: a UA obeys ONLY its most-specific group — these groups do NOT
# inherit the global "User-agent: *" rules, so each must carry its own
# Disallow: /api/. Social-preview bot groups above intentionally keep full
# access — they must fetch /api/og images.
User-agent: ChatGPT-User
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: OAI-SearchBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Claude-User
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Claude-SearchBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: PerplexityBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Amazonbot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: YouBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: cohere-ai
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

# Agent-category fetchers (a person's assistant fetching a page at answer
# time — the citation itself). Explicit groups so the policy is legible to
# Cloudflare's 2026-09-15 Search/Agent/Training split; same retrieval rules.
User-agent: Perplexity-User
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: meta-externalfetcher
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: DuckAssistBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: MistralAI-User
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Applebot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
# Applebot is a real search engine with a rich-result surface (Apple Search,
# Siri, Spotlight), and robots groups do NOT cascade — so the /api/logo/
# exception in the wildcard group below does not reach it. Mirror it here or
# Apple keeps seeing a page that promises an image and blocks every one.
# The AI-answer groups deliberately do not get this:
# they consume text, and each extra allowance is extra crawl load for us.
Allow: /api/logo/
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=
# Groups do not cascade (RFC 9309), so the /locales/ rule in the wildcard group
# below never reaches Applebot — mirror it here. Applebot receives server-side
# rendered pages and loses nothing without the locale JSON files.
Disallow: /locales/

# AI training bots — allowed by explicit, documented owner decisions
# (OpenAI/Anthropic/Google 2026-06-10; Meta AI + Common Crawl 2026-09-03: the
# goal is to be known to the models, and Cloudflare's 2026-09-15 policy makes
# a "Training" block sweep up Googlebot/Bingbot/Applebot too). Apple-Extended,
# FacebookExternalHit-AI, Bytespider (outside /zh), Diffbot, ImagesiftBot and
# Omgili below remain blocked.
User-agent: GPTBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: ClaudeBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: anthropic-ai
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Google-Extended
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: FacebookExternalHit-AI
Disallow: /

# Bytespider (ByteDance/TikTok + Doubao LLM): only allowed on /zh/* surface.
# Doubao serves the Chinese market, so the Chinese-language pages are the only
# surface where a citation can reach its audience. Confining Bytespider there
# keeps that citation surface alive while avoiding origin-heavy crawl traffic
# across the other locales.
User-agent: Bytespider
Allow: /zh/
Disallow: /

User-agent: CCBot
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt
Disallow: /api/
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgili
Disallow: /

# ── Shared crawl policy: bingbot + all other crawlers ─────────────────
# Bing powers ChatGPT search and Microsoft Copilot, so bingbot carries an
# explicit group (added 2026-09-01). RFC 9309 groups do not cascade — an
# explicit group means bingbot ignores the wildcard group — so both groups
# below state the identical full rule set, generated from one source.
#
# Feed endpoints (/feed.xml, /feed/*) were unblocked 2026-09-01: the feed
# now serves per-item summaries instead of 456 KB of full article bodies,
# and RSS is a discovery channel for Bing and AI systems.

# Content signals are declared twice, from ONE constant: the Content-Signal
# HTTP response header on every page, and the inline Content-Signal line at
# the head of the bingbot and * groups below (contentsignals.org directive;
# strict RFC-9309 validators flag it — accepted, owner decision 2026-09-03):
#   search=yes    → permit building search indexes and short excerpts
#   ai-input=yes  → permit RAG / grounding at query time
#   ai-train=yes  → permit training
# Retrieval, agent AND training bots (OpenAI/Anthropic/Google/Meta/Common
# Crawl) are allowed above; Apple-Extended, Bytespider (outside /zh), Diffbot,
# ImagesiftBot and Omgili stay blocked.

# Crawlable OG cards — allowed 2026-08-15, closed 2026-08-25
# (measured crawl-budget decision).
#
# The 2026-08-15 change moved the stock share card out of /api/ to
# /og/stock/{version}/{TICKER}.png so the "Disallow: /api/" below stayed
# intact. That worked, and it also handed Googlebot the same bill the
# /api/og/ Allow had run up in June — the card generator was the problem, not
# the path it sat on.
#
# Measured over 8 days of VERIFIED Googlebot (reverse-DNS confirmed, so this
# is not the 24% of Googlebot-UA traffic that is our own tooling):
#   /og/stock/*   12,085 requests = 16.8% of the entire crawl budget,
#                 across 6,011 distinct images, exactly 2.0 fetches each.
#                 At 163 KB per card that is 1.9 GB in 8 days — and Google
#                 has reached only 6,011 of ~36.8K tickers, so the finished
#                 set is ~72K requests and ~11 GB.
# Against that: zero clicks from images in 90 days, and the pages these cards
# belong to average position 58. A thumbnail on page 6 buys nothing; the
# requests it costs are requests not spent on the tens of thousands of URLs
# sitting in "Discovered - currently not indexed".
#
# Social preview bots are UNAFFECTED — Twitterbot, facebookexternalhit,
# LinkedInBot, Slackbot, Discordbot, TelegramBot and WhatsApp each have their
# own "Allow: /" group above, and RFC 9309 groups do not cascade, so this
# Disallow never reaches them. og:image keeps working everywhere a human
# actually sees it. Applebot likewise keeps its own group untouched.
#
# /api/logo/ below stays ALLOWED and that is deliberate, not an oversight:
# logos are ~2 KB against the card's 163 KB, and they are the only fetchable
# image left on a stock page, so "max-image-preview:large" still refers to
# something Google is permitted to read.
#
# Do NOT re-open this without measuring the crawl split first. It has now been
# opened and closed twice on the same reasoning (2026-06-09 /api/og/,
# 2026-08-15 /og/) — the third time will cost another quarter.

# API endpoints stay blocked — crawlers should only index HTML pages, not JSON responses.
# /api/og/ exception removed 2026-06-09: the Allow let
# Googlebot spend ~16% of its sampled crawl budget (37K fetches/7d) on OG
# share-card renders that carry X-Robots-Tag: noindex anyway. Social preview
# bots (Twitterbot, facebookexternalhit, LinkedInBot, Slackbot, Discordbot,
# TelegramBot, WhatsApp) keep og:image access via their explicit Allow: /
# groups above — robots.txt groups don't cascade, so this Disallow never
# applies to them. Do not re-add the Allow.
#
# /api/logo/ exception: logo images referenced by pages must stay fetchable.
# Every image on a stock page lived under /api/ — og:image, twitter:image,
# Article.image and the only content <img> — so a page declaring
# max-image-preview:large had not one image Google was permitted to fetch.
# No thumbnail beside our result, no Discover eligibility.
#
# Only the LOGO path is reopened, and it is nothing like the card generator
# that caused the 2026-06-09 problem: logos are 0.7-11 KB, served from a
# 7-day process cache behind a 30-day edge cache, and there are ~36.8K of them
# total rather than one render per request. The expensive /api/og/ generator
# stays blocked. Allow wins over Disallow on the longer path (RFC 9309).

# i18n JSON bundles — /locales/ added 2026-08-15 (crawl-budget
# measurement). 4,436 files sit under this one prefix: 10 full bundles (up to 2.0 MB
# each), 10 core bundles, 4,416 per-namespace chunks. They are served by
# express.static (server/app.ts:588) and carry no X-Robots-Tag.
#
# Nothing links to them: no <link>, no sitemap entry, no href in any HTML. They
# are reached ONLY by executing our JS — and the bot that executes it gains
# nothing, because bots receive server-rendered pages and never mount the app
# for bot UAs that received #ssr-content. The indexed DOM is 100% server HTML,
# so the i18n store never reaches it. The requests are still fired because
# client/src/i18n/config.ts calls initI18n() at module scope.
#
# Measured on production with a real Chrome under a Googlebot UA, aborting every
# /locales/ request: /stock/aapl fired 20, /tr/stock/aapl 45, /earnings 6,
# /product 5 (one being the 1.75 MB full bundle) — and the rendered text came
# back byte-identical each time, with zero raw i18n keys. The same probe under a
# human UA DID break (5 raw keys on /stock/aapl), which is what proves the bot
# result is a real negative rather than a blind probe.
#
# X-Robots-Tag: noindex is NOT a substitute — Google must keep crawling a URL to
# see a noindex, so the fetch waste would continue.

User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt

# Markdown mirrors for AI systems (MD Mirror — AEO). Search engines index
# the HTML version (canonical) — .md responses return X-Robots-Tag: noindex.
Allow: /stock/*.md
Allow: /etf/*.md
Allow: /journal/*.md
# Multi-language markdown mirrors (coverage-gated per ticker — 404 when no translation)
Allow: /tr/stock/*.md
Allow: /es/stock/*.md
Allow: /de/stock/*.md
Allow: /fr/stock/*.md
Allow: /pt/stock/*.md
Allow: /ja/stock/*.md
Allow: /ko/stock/*.md
Allow: /zh/stock/*.md

# Prevent crawling tabbed stock page variants (canonical is /stock/ticker without ?tab=)
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=

# Client-side recovery params (redirect-loop bait for strict crawlers)
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

# Private/auth-only pages
Disallow: /admin
Disallow: /settings
Disallow: /account
Disallow: /dashboard
Disallow: /portfolio-tracker
# $ end-anchor (RFC 9309): bare "/portfolio" prefix-blocked the PUBLIC,
# sitemap-submitted /portfolio-review and /portfolio-risk-guide pages
Disallow: /portfolio$
Disallow: /portfolio/
Disallow: /portfolio-analytics
Disallow: /war-room
Disallow: /login
Disallow: /signup
Disallow: /register
Disallow: /auth/
Allow: /author/
Disallow: /onboarding
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /profile
Disallow: /messages
Disallow: /trading
Disallow: /order-history
Disallow: /funding
Disallow: /email-preferences
Disallow: /watchlist
Disallow: /hot-picks
Disallow: /agent-ops

# OG share cards — closed 2026-08-25 after crawl-budget measurement
# (full history in the note above the group blocks).
Disallow: /og/

# API endpoints are JSON, not pages. /api/logo/ stays fetchable — the only
# image on a stock page crawlers are permitted to read (longer-path Allow
# wins per RFC 9309). Full history above the group blocks.
Allow: /api/logo/
Disallow: /api/

# i18n JSON bundles — JS-only fetches, nothing links to them (measured
# 2026-08-15; full note above the group blocks).
Disallow: /locales/

# Asset directories
Disallow: /attached_assets/

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /ai/
Allow: /.well-known/ai.txt

# Markdown mirrors for AI systems (MD Mirror — AEO). Search engines index
# the HTML version (canonical) — .md responses return X-Robots-Tag: noindex.
Allow: /stock/*.md
Allow: /etf/*.md
Allow: /journal/*.md
# Multi-language markdown mirrors (coverage-gated per ticker — 404 when no translation)
Allow: /tr/stock/*.md
Allow: /es/stock/*.md
Allow: /de/stock/*.md
Allow: /fr/stock/*.md
Allow: /pt/stock/*.md
Allow: /ja/stock/*.md
Allow: /ko/stock/*.md
Allow: /zh/stock/*.md

# Prevent crawling tabbed stock page variants (canonical is /stock/ticker without ?tab=)
Disallow: /stock/*?tab=
Disallow: /stock/*&tab=

# Client-side recovery params (redirect-loop bait for strict crawlers)
Disallow: /*?_reload=
Disallow: /*&_reload=
Disallow: /*?_v=
Disallow: /*&_v=

# Private/auth-only pages
Disallow: /admin
Disallow: /settings
Disallow: /account
Disallow: /dashboard
Disallow: /portfolio-tracker
# $ end-anchor (RFC 9309): bare "/portfolio" prefix-blocked the PUBLIC,
# sitemap-submitted /portfolio-review and /portfolio-risk-guide pages
Disallow: /portfolio$
Disallow: /portfolio/
Disallow: /portfolio-analytics
Disallow: /war-room
Disallow: /login
Disallow: /signup
Disallow: /register
Disallow: /auth/
Allow: /author/
Disallow: /onboarding
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /profile
Disallow: /messages
Disallow: /trading
Disallow: /order-history
Disallow: /funding
Disallow: /email-preferences
Disallow: /watchlist
Disallow: /hot-picks
Disallow: /agent-ops

# OG share cards — closed 2026-08-25 after crawl-budget measurement
# (full history in the note above the group blocks).
Disallow: /og/

# API endpoints are JSON, not pages. /api/logo/ stays fetchable — the only
# image on a stock page crawlers are permitted to read (longer-path Allow
# wins per RFC 9309). Full history above the group blocks.
Allow: /api/logo/
Disallow: /api/

# i18n JSON bundles — JS-only fetches, nothing links to them (measured
# 2026-08-15; full note above the group blocks).
Disallow: /locales/

# Asset directories
Disallow: /attached_assets/

Sitemap: https://www.stockexpertai.com/sitemap.xml

# Locale stock sitemaps — one per active language so Google discovers
# translated stock pages directly instead of crawling from internal links.
Sitemap: https://www.stockexpertai.com/sitemap-ja-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-ko-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-de-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-es-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-zh-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-fr-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-pt-stocks.xml
Sitemap: https://www.stockexpertai.com/sitemap-tr-stocks.xml

# The markdown-mirror sitemaps (sitemap-stocks-md.xml, sitemap-etfs-md.xml,
# sitemap-hubs-md.xml) are deliberately NOT advertised here (2026-08-20
# measurement): every .md URL answers X-Robots-Tag:
# noindex, so listing 37,283 of them in a Sitemap: directive told Google to
# process a sitemap made entirely of non-indexable duplicates. The .md routes
# and sitemap endpoints stay live for AI crawlers (GPTBot/bingbot/Amazonbot
# consume them at scale); discovery runs through /llms.txt, /sitemap.md,
# <link rel="alternate" type="text/markdown"> and Accept: text/markdown.

# Earnings POC sitemaps — quarterly earnings preview pages.
# Decoupled from translation_languages.is_active: earnings POC ships in all
# 9 supported langs at once (i18n bypass — single shared bundle). Listing
# every lang here lets Google discover translated earnings pages even
# before the cloner marks the language "active".
Sitemap: https://www.stockexpertai.com/sitemap-earnings.xml
Sitemap: https://www.stockexpertai.com/sitemap-earnings-tr.xml
Sitemap: https://www.stockexpertai.com/sitemap-earnings-de.xml
Sitemap: https://www.stockexpertai.com/sitemap-earnings-fr.xml
Sitemap: https://www.stockexpertai.com/sitemap-earnings-ko.xml
Sitemap: https://www.stockexpertai.com/sitemap-earnings-zh.xml

# Note: /sitemap.md is NOT advertised here. The sitemaps.org protocol
# expects XML-formatted sitemaps in the Sitemap: directive, and Google
# Search Console flags non-XML entries as invalid. AI agents discover
# /sitemap.md through the "Markdown Sitemap" section in /llms.txt, which
# is the correct channel for markdown-formatted discovery resources.

# AI/LLM discovery (see https://llmstxt.org)
# LLMs-txt and LLMs-full-txt are non-standard directives that cause
# Lighthouse to flag robots.txt as invalid. AI bots discover these
# via /.well-known/llms.txt or <link> tags instead.
# LLMs-txt: https://www.stockexpertai.com/llms.txt
# LLMs-full-txt: https://www.stockexpertai.com/llms-full.txt