# WFA Digital — https://wfadigital.com
# robots.txt

# Standard search engines
User-agent: *
Allow: /
Allow: /jobs/
Allow: /jobs
# /api/ stays closed, with two exceptions. Both are read-only JSON that a
# public page needs in order to render, and blocking them made Googlebot render
# the research reports as 404s — a page that fetches its own body cannot be
# denied that fetch.
Allow: /api/research/body
Allow: /api/micro1/body
Allow: /api/world-map.json
# The pages fetch their data from /api/trpc after they load. With that path
# blocked, Googlebot's renderer painted "Company not found", "This page did not
# load" and "Story not found" on company, job and article pages while the real
# text sat in a hidden block (measured 2026-09-12 with a robots-obeying
# headless Chrome). Google must be able to render what a reader sees.
Allow: /api/trpc/
# The job feeds advertised 200 posting URLs to crawlers on every visit. They
# stay for feed readers; crawlers have the hubs.
Disallow: /feed/
# Referral redirects. Never crawl these: each hit writes an outbound-click row
# and ClaudeBot walked all 206 in eight minutes, which is 79% of the table.
Disallow: /go/
Disallow: /api/
Disallow: /dashboard
Disallow: /jobs-dashboard
Disallow: /advertiser/dashboard
# Localized hubs — gated while in build. Remove a prefix when its hub launches.
Allow: /es/trabajo-remoto/espana
Disallow: /es/
Allow: /pt/micro1
Disallow: /pt/
Disallow: /pl/

# AI crawlers are welcome and are covered by the rules above.
#
# They used to have one group each, containing only "Allow: /". Under RFC 9309 a
# named group replaces the "*" group entirely, so those 13 groups quietly
# cancelled every Disallow for exactly the crawlers that fetch hardest: measured
# on 2026-09-20, meta-externalagent had walked /go/ 37 times and Applebot had
# made 11,762 requests to /api/trpc. Removing the groups keeps them allowed on
# the site and closes /go/, /api/ and the dashboards to them as well.
# Welcome: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-User,
# Claude-SearchBot, anthropic-ai, PerplexityBot, Perplexity-User, YouBot,
# cohere-ai, Applebot, Amazonbot, meta-externalagent, Bytespider, CCBot.

# Third-party SEO/backlink crawlers — no value to us, pure server load
User-agent: SERankingBacklinksBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps
Sitemap: https://wfadigital.com/sitemap.xml