# lineaum.com
# Primary canonical domain: https://lineaum.com
#
# Policy:
#  - Allow legitimate search and AI-search crawlers.
#  - Allow AI assistant + AI training crawlers (ChatGPT, Gemini, Claude, Perplexity).
#  - Block /admin/ and /api/. /auth uses meta noindex so Google can honor it.
# Content Signals (https://contentsignals.org/) are declared inside each group.

# --- Legitimate search + AI search: allow everything ---
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- AI assistants + AI training crawlers: allow ---
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: YouBot
Allow: /

# --- Low-value scrapers: disallow ---
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /


# --- Default policy for everything else ---
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /admin/
Disallow: /api/
# /auth is served with a noindex meta tag instead of a robots block, so Google
# can crawl it and honor noindex (avoids "Indexed, though blocked by robots.txt").

# Indexability is per-route × per-locale (see ROUTE_LOCALE_INDEXABILITY).
# Pages outside the matrix render with noindex,follow until translated.

Sitemap: https://lineaum.com/sitemap.xml