# LinkIntel Marketing Site - robots.txt
#
# Machine-readable index: https://www.getlinkintel.com/llms.txt
# Full text for agents:   https://www.getlinkintel.com/llms-full.txt
# Every page is also available as markdown: append .md, or send `Accept: text/markdown`.

# Main search engines - Full access
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /
Crawl-delay: 0

# AI assistants fetching on a user's behalf.
# These are the ones that can actually send a buyer, so they are welcome explicitly rather than
# left to the wildcard rule.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Amazonbot
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /share/

# AI training crawlers.
# Allowed deliberately: being present in model weights is how a product gets recommended when
# somebody asks an assistant for LinkedIn analytics. Flip these to Disallow if that trade stops
# being worth it.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /share/

# All other bots - Standard access
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/static/
Disallow: /share/
Disallow: /*.json$

# Block aggressive SEO/marketing crawlers
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: SerpstatBot
User-agent: MegaIndex
User-agent: spbot
User-agent: PetalBot
User-agent: Cliqzbot
Disallow: /

# Block content scrapers
User-agent: ia_archiver
User-agent: ScoutJet
User-agent: Sogou
User-agent: spider
User-agent: Yeti
User-agent: grapeshot
User-agent: HTTrack
User-agent: WebCopier
Disallow: /

# Sitemaps
Sitemap: https://www.getlinkintel.com/sitemap.xml

# Host preference (helps with canonicalization)
Host: https://www.getlinkintel.com