# AI Hentai Chat - public crawler access policy
# Canonical publication origin: https://ai-chat-hentai.com
# Policy updated: 2026-10-07
# Public pages, images, styles, scripts and machine-readable files are crawlable.
# Search, AI search, user-directed retrieval and documented training agents are allowed.
# Allow is a crawl preference, not a guarantee of indexing, citation or model inclusion.
# Individual page metadata still determines whether a page is indexable.
# No Crawl-delay is imposed and no private filesystem paths are exposed here.
# The wildcard group also permits agents not individually listed below.

# Default access for every crawler
User-agent: *
Allow: /
Disallow:

# Google - Google Search crawling and discovery
User-agent: Googlebot
Allow: /
Disallow:

# Google - Gemini training and grounding usage control; no separate HTTP crawler
User-agent: Google-Extended
Allow: /
Disallow:

# Google - Crawls requested by site owners for Vertex AI Agents
User-agent: Google-CloudVertexBot
Allow: /
Disallow:

# Microsoft - Bing search indexing
User-agent: bingbot
Allow: /
Disallow:

# OpenAI - ChatGPT Search discovery and indexing
User-agent: OAI-SearchBot
Allow: /
Disallow:

# OpenAI - User-initiated retrieval; robots rules may not apply to these actions
User-agent: ChatGPT-User
Allow: /
Disallow:

# OpenAI - Content collection that may support foundation-model training
User-agent: GPTBot
Allow: /
Disallow:

# OpenAI - Validation of pages submitted as ads; this permission implies no submission
User-agent: OAI-AdsBot
Allow: /
Disallow:

# Anthropic - Search indexing for relevant Claude responses
User-agent: Claude-SearchBot
Allow: /
Disallow:

# Anthropic - User-initiated content retrieval
User-agent: Claude-User
Allow: /
Disallow:

# Anthropic - Content collection that may contribute to model training
User-agent: ClaudeBot
Allow: /
Disallow:

# Perplexity - Search discovery; operator states this crawler is not for foundation-model training
User-agent: PerplexityBot
Allow: /
Disallow:

# Perplexity - User-initiated retrieval; operator says these requests generally ignore robots rules
User-agent: Perplexity-User
Allow: /
Disallow:

# Apple - Apple search and contextual content retrieval
User-agent: Applebot
Allow: /
Disallow:

# Apple - Foundation-model training usage control; this token does not crawl pages
User-agent: Applebot-Extended
Allow: /
Disallow:

# Amazon - Content collection for products and services; may support AI model training
User-agent: Amazonbot
Allow: /
Disallow:

# Amazon - Search experiences, including Alexa; not foundation-model training
User-agent: Amzn-SearchBot
Allow: /
Disallow:

# Amazon - User-initiated live retrieval; may not follow all robots directives
User-agent: Amzn-User
Allow: /
Disallow:

# DuckDuckGo - Real-time retrieval for AI-assisted answers; not AI model training
User-agent: DuckAssistBot
Allow: /
Disallow:

# Mistral - User-initiated retrieval
User-agent: MistralAI-User
Allow: /
Disallow:

# Mistral - Automated search indexing; not generative AI training
User-agent: MistralAI-Index
Allow: /
Disallow:

# Mistral - Content collection for generative AI training datasets
User-agent: MistralAI-Training
Allow: /
Disallow:

# Common Crawl - Public web collection; does not require JavaScript or cookies
User-agent: CCBot
Allow: /
Disallow:

# Diffbot - Structured web extraction; empty Disallow is included because this crawler ignores Allow
User-agent: Diffbot
Allow: /
Disallow:

# Meta - AI crawler identity documented in Cloudflare's crawler reference
User-agent: meta-externalagent
Allow: /
Disallow:

# Meta - AI assistant identity documented in Cloudflare's crawler reference
User-agent: meta-externalfetcher
Allow: /
Disallow:

# Meta - AI crawler identity documented in Cloudflare's crawler reference
User-agent: FacebookBot
Allow: /
Disallow:

# ByteDance - AI crawler identity documented in Cloudflare's crawler reference
User-agent: Bytespider
Allow: /
Disallow:

# Canonical indexable HTML inventory
Sitemap: https://ai-chat-hentai.com/sitemap.xml

# Machine-readable discovery: /llms.txt and /llms-full.txt
# Editorial update feeds: /feed.xml (RSS) and /feed.atom (Atom)
# Publication and security contacts: /humans.txt and /.well-known/security.txt