# A crawler obeys only the most specific User-agent group that matches it and
# ignores every other group, including "*". So each named bot below repeats the
# Disallow rules. Without that, every named AI crawler was free to crawl /admin/
# while only unnamed bots were blocked.

# Deliberately no `Disallow: /*?` here. Every page emits a canonical, and a
# robots block on parameterised URLs would stop crawlers reading that canonical,
# leaving UTM-tagged variants indexable-but-uncrawlable, which is worse than
# letting them be crawled and folded into the canonical. At 30 static pages
# crawl budget is not a constraint worth trading that for.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/

# --- AI assistants and answer engines -------------------------------------
# Explicitly welcome to crawl and cite this site.

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /api/

Sitemap: https://marketivv.com/sitemap-index.xml