# == Default (Global Rules for All Standard & Search Bots) ==
User-agent: *
Allow: /
Allow: /_api/mcp
Disallow: /_api/*
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*
Disallow: /*?lightbox=
Disallow: /profile/
Disallow: /*/profile/
Disallow: /account/

# == AI, LLM & RAG Systems (All Co-Pilots & Agents Grouped) ==
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Cohere-ai
User-agent: Bytespider
User-agent: CCBot
Allow: /
Allow: /_api/mcp
Disallow: /_api/*

# Google Ads Bots (Needs broad access to verify landing pages)
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*

# == Crawl Delay for Heavy SEO Bots ==
User-agent: dotbot
User-agent: AhrefsBot
Crawl-delay: 10

# == Blocked Spam, Aggressive Scrapers & Resource Hogs ==
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: MegaIndex.ru
User-agent: ZoominfoBot
User-agent: SerpstatBot
User-agent: SpyonWeb
User-agent: ExtLinksBot
User-agent: aiHitBot
User-agent: SMTBot
User-agent: Rogerbot
User-agent: Exabot
Disallow: /

Sitemap: https://www.archivinci.com/sitemap.xml
Sitemap: https://www.archivinci.com/_functions/multilingualSitemap