User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Allow: /rss.xml

# Content files for AI systems and LLMs
#   Index:        https://www.incorpx.io/llms.txt
#   Any page also returns Markdown to a request carrying: Accept: text/markdown
Allow: /llms.txt

# Throttle SEO tool crawlers (high request volume)
User-agent: AhrefsBot
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 30

User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 30

User-agent: BLEXBot
Crawl-delay: 30

User-agent: DataForSeoBot
Crawl-delay: 30

# Throttle secondary search engines
User-agent: Baiduspider
Crawl-delay: 15

User-agent: Baiduspider-render
Crawl-delay: 15

User-agent: YandexBot
Crawl-delay: 15

User-agent: PetalBot
Crawl-delay: 15

# Throttle social media crawlers
User-agent: facebookexternalhit
Crawl-delay: 10

User-agent: meta-webindexer
Crawl-delay: 10

User-agent: TikTokSpider
Crawl-delay: 15

Sitemap: https://www.incorpx.io/sitemap.xml
Sitemap: https://www.incorpx.io/variants-sitemap.xml