# InsiderAlpha robots.txt
# Allow major search engines to crawl public marketing/content pages.
# Disallow personal/authenticated app surfaces and API endpoints.
#
# Editing this file? Copy it byte for byte to edge/robots-fallback.txt in the
# same commit. The edge router serves that copy whenever this server cannot
# answer, because Google reads a 5xx on robots.txt as "stop crawling the whole
# site" for about 12 hours. public-frontend/tests/edgeRobotsFallback.test.ts
# fails while the two files differ.

User-agent: *
Allow: /

# Authenticated personal areas (no value to search index)
Disallow: /my-trades
Disallow: /performance
Disallow: /portfolio
Disallow: /settings
Disallow: /profile
Disallow: /brokers
Disallow: /trading-plan
Disallow: /trade-picker
Disallow: /mcp

# /subscription-required and the other auth-adjacent PAGES are intentionally
# NOT disallowed here: they serve <meta name="robots" content="noindex">, and
# Google must be allowed to CRAWL a URL to SEE its noindex and drop it from
# the index. A robots.txt Disallow would hide the noindex and let bare URLs
# linger indexed.
#
# /login is the exception, and is disallowed (2026-08-31). Until 2026-10-10 it
# was not a page: outside the rare Auth0-error case it answered a 307 into
# /auth/login, the same character as the /auth/ endpoints below. It is now
# the sign-in chooser, noindex like /register, and stays disallowed for the
# reasons that follow, which have not changed. The header links /login on every page, and
# between 2026-08-27 and 2026-09-16 the signed-out asks on every insider page
# also minted a UNIQUE /login?returnTo={path}, one redirecting URL per
# document: Ahrefs' 2026-08-30 crawl flagged 1,438 pages "links to redirect"
# for them and Search Console's "Blocked by robots.txt" bucket reached 41,601
# URLs. From the deploy of the 2026-09-16 change on, no document carries such
# a link: a destination that names the page is attached when the reader
# clicks and is never rendered as
# an href (public-frontend/components/AccountLink.tsx). This rule stays for
# the header link and for the minted URLs Google already holds; it keeps
# listing them until it recrawls their sources. Never press Validate Fix on
# that bucket, it fails by design while these Disallows exist:
# docs/seo/SEARCH_CONSOLE_ROBOTS_BUCKET.md.
Disallow: /login

# Cloudflare's infrastructure path. Email Address Obfuscation rewrites the
# footer's mailto into /cdn-cgi/l/email-protection#..., which answers 404 when
# fetched as a document; Cloudflare's own guidance is to disallow /cdn-cgi/ so
# crawlers do not report it as a sitewide broken link.
Disallow: /cdn-cgi/

# Backend API: no SEO value, can be expensive to crawl
Disallow: /api/

# The one API document that IS meant to be read by a machine on a reader's
# behalf. /api/pricing is anonymous JSON carrying only the public price
# ladder, the same figures /pricing renders, and llms.txt and llms-full.txt
# both send assistants there for the live prices. Longest match wins, so this
# reopens exactly that path and nothing else under /api/. The rule is repeated
# in every AI stanza below, because each of those groups is read on its own.
Allow: /api/pricing

# The Next server-side data proxy. Same character as /api/ - it returns JSON
# for the authenticated app, never a document.
Disallow: /bff/

# OAuth handshake endpoints. Unlike the SPA auth PAGES above, these are pure
# redirects with no body, so there is no noindex for a crawler to discover and
# nothing to keep out of the index; the only effect of crawling them is wasted
# budget bouncing off the identity provider.
Disallow: /auth/

# /share/* is a backend-owned path that no longer backs any page on this site.
# Nothing in public-frontend/ calls it: Next reaches the backend over the Swarm
# network via BACKEND_URL, never over this public path. Two things still live
# there, both driven from outside the frontend: /share/{ticker}, which is now a
# permanent 301 to /company/{ticker} kept alive for alert links frozen in
# Discord channel history, and /share/insider/{slug}, the last server-rendered
# page in the stack. It stays disallowed because the canonical URLs are
# /company/{ticker}, /insider/{name-slug}-{id} and /learn/{slug}, and indexing
# the raw source of those pages would be duplicate content. The insider shape
# is the whole slug, not the bare id: /insider/{id} answers a 301 to it, so a
# comment naming the short form describes a redirect rather than a canonical.
Disallow: /share/

# PostHog reverse-proxy paths — analytics endpoints, not page content. Bots
# that render JS (meta-externalagent, etc.) hammer these for no SEO value.
Disallow: /panda/

# Next.js React Server Component payloads. The app router fetches the next
# page's data as /{path}?_rsc={hash}, with an `RSC: 1` request header, when a
# visitor hovers or is about to click a link. A crawler that finds those URLs
# in a rendered page and fetches them WITHOUT the header gets the ordinary HTML
# document back: one more copy of a page it already has, under a URL that
# differs only by a cache-busting hash. Measured 2026-10-12: 96% of
# OAI-SearchBot's fetches were ?_rsc= variants. Nothing a crawler needs lives
# behind the query: Google's renderer reads the page HTML and follows <a href>,
# never the prefetch, and a browser is not a robot and is unaffected. `*` is
# the wildcard Google, Bing and the AI fetchers all document, so this closes
# the variant on every path at once. Repeated in each AI stanza below, because
# a crawler reads only its own group.
Disallow: /*?_rsc=

# Facebook / Meta external agent: extremely heavy crawler that fully renders
# the SPA (includes PostHog flag/config requests). Provides no SEO value to
# our product and accounted for ~50% of bot traffic in May 2026. Restrict to
# the OG share surface only.
#
# The two template sets are named explicitly even though `Disallow: /` already
# covers them. Longest match wins, so if anyone ever softens the default here
# the way `facebookexternalhit` below had been softened, /insider/ and
# /company/ stay closed instead of quietly reopening the 53,000 URLs this
# stanza exists to keep Meta off.
User-agent: meta-externalagent
Disallow: /
Disallow: /insider/
Disallow: /company/
Allow: /share/
Allow: /og-image.png
Allow: /favicon.ico
Allow: /favicon.svg

User-agent: meta-externalfetcher
Disallow: /
Disallow: /insider/
Disallow: /company/
Allow: /share/
Allow: /og-image.png
Allow: /favicon.ico
Allow: /favicon.svg

# meta-webindexer is the newest name in the same Meta crawl fleet (2026-10-12:
# 28.7% of origin traffic, and it had never once fetched this file, so it was
# crawling under no rules at all). Same job as the two stanzas above, so the
# same rules: off the two heavy templates, the OG share surface open. Like
# them, this is intent; the enforcing layer is the Cloudflare rate-limit rule
# on AS32934.
User-agent: meta-webindexer
Disallow: /
Disallow: /insider/
Disallow: /company/
Allow: /share/
Allow: /og-image.png
Allow: /favicon.ico
Allow: /favicon.svg

# facebookexternalhit is Meta's LINK-PREVIEW fetcher, and it is deliberately
# NOT treated like the two stanzas above. It is not part of a crawl: it makes
# one request, when a PERSON pastes an InsiderAlpha URL into WhatsApp,
# Messenger or a Facebook post, so the chat renders a title, a description and
# a card instead of a naked link. The crawl fleet sweeps URLs nobody asked
# for; this fetcher only ever asks for a page someone already chose to share.
#
# It was briefly given Disallow: /company/, /insider/ and /share/ on
# 2026-08-29 on the theory that it was the same fleet. It is not, and that
# bought nothing. The sweep that pushed the origin over was
# meta-externalagent plus raw AS32934 address ranges (measured 2026-08-29:
# 45/45 clean responses at concurrency 3 against 41.7% 5xx at concurrency 8,
# with the database pool logging 30/30 connections checked out in the same
# window), robots.txt is ADVISORY in any case, and the layer that enforces
# the limit is a Cloudflare rate-limit rule on AS32934 that lives in the
# dashboard and not in this repository. What the change did cost was the
# preview on every shared company and insider link.
#
# So: documents open, endpoints closed. The Disallows below are endpoints and
# not pages, and nothing an unfurler does needs the JSON API, the server-side
# data proxy, the OAuth redirects or the analytics proxy.
#
# /share/ is open HERE and nowhere else among the Meta stanzas' siblings.
# /share/{ticker} is a 301 into /company/{ticker} kept alive for alert links
# frozen in Discord channel history and /share/insider/{slug} is the last
# server-rendered page in the stack, so /share/ is the surface built for
# exactly this fetcher. It stays shut to indexers because it is duplicate
# content, and a preview fetcher indexes nothing.
#
# The four Allow lines are redundant against `Allow: /` and are here on
# purpose. Longest match wins and a tie goes to Allow, so a later
# "consistency fix" that pastes the crawl fleet's Disallows into this stanza
# cannot quietly take the previews down a second time. If you arrived here
# wanting to close these paths, the thing you actually want is the WAF rule.
User-agent: facebookexternalhit
Allow: /
Allow: /company/
Allow: /insider/
Allow: /learn/
Allow: /share/
Disallow: /api/
Disallow: /bff/
Disallow: /auth/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/

# FacebookBot is Meta's bulk crawler for AI training, NOT the link-preview
# fetcher above; this comment used to call it "link-preview / AI-training",
# and that conflation is how the preview fetcher ended up with a crawler's
# rules. It is therefore treated as what it is, and identically to every
# other AI crawler in this file (GPTBot, ClaudeBot, CCBot and the rest):
# public content open, /share/ closed as duplicate content, endpoints closed.
# It has never appeared in the load forensics, and shutting one more AI
# crawler out of the templates would be the withholding lever the rest of
# this file exists to warn about. Revisit only if the traffic actually moves
# here, and revisit it in the WAF first.
User-agent: FacebookBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

# ---------------------------------------------------------------------------
# AI crawlers (LLM training + answer-engine bots).
# Explicit allow-rules so they index public content but stay off /api/, /share/
# (canonical URL is /company/*, /insider/* etc.) and analytics endpoints.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Google-Extended
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Amazonbot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Bytespider
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: CCBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

# Answer-engine fetchers added 2026-10-12. Each is the user-triggered or
# search-index half of a vendor whose training crawler already has a stanza
# above (Claude-User and Claude-SearchBot beside ClaudeBot, MistralAI-User for
# Le Chat, DuckAssistBot for DuckDuckGo's AI answers). They read only their own
# group, never `*`, so without one of their own they would inherit nothing
# here. Same rules as every other AI stanza: public content open, /share/
# closed as duplicate content, endpoints closed, the public price ladder open.
User-agent: Claude-User
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Allow: /api/pricing
Disallow: /bff/
Disallow: /auth/
Disallow: /share/
Disallow: /panda/
Disallow: /login
Disallow: /cdn-cgi/
Disallow: /*?_rsc=

# Sitemap (dynamically generated by the backend - sitemap index)
Sitemap: https://insideralpha.ai/sitemap.xml

# LLM guidance (llms.txt standard): https://insideralpha.ai/llms.txt