# robots.txt for SAUTER Pianofortemanufaktur
# https://www.sauter-pianos.de/
#
# Goal:
# - Allow public German (/de/) and English (/en/) content to be crawled.
# - Allow major search engines and AI search / answer crawlers.
# - Keep Drupal administration, internal search and account functions out of crawl.
#
# Last reviewed: 2026-08-27

# --- OpenAI / ChatGPT Search ---
User-agent: OAI-SearchBot
Allow: /

# OpenAI crawler used for model training.
# Keep "Allow: /" only if SAUTER intends to permit this use.
User-agent: GPTBot
Allow: /

# User-initiated ChatGPT page access.
User-agent: ChatGPT-User
Allow: /

# --- Anthropic / Claude ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- Google / Gemini discovery via Google Search ---
User-agent: Googlebot
Allow: /

# Google-Extended controls certain generative-AI uses separately from Google Search.
# Keep this allowed if SAUTER wants to maximize AI accessibility.
User-agent: Google-Extended
Allow: /

# --- Microsoft / Bing / Copilot ---
User-agent: Bingbot
Allow: /

# --- Common Crawl ---
# Common Crawl data is used by many research and AI systems.
# Keep this allowed if broad LLM discoverability is desired.
User-agent: CCBot
Allow: /

# --- General crawler rules ---
User-agent: *

# Drupal administration and account areas.
Disallow: /admin/
Disallow: /de/admin/
Disallow: /en/admin/

Disallow: /user/
Disallow: /de/user/
Disallow: /en/user/

# Drupal content creation / editing endpoints.
Disallow: /node/add/
Disallow: /de/node/add/
Disallow: /en/node/add/

# Internal site search result pages.
Disallow: /search/
Disallow: /de/search/
Disallow: /en/search/

# Drupal comment / filter utility endpoints.
Disallow: /comment/reply/
Disallow: /filter/tips

# Media oEmbed endpoints are machine utilities, not editorial content.
Disallow: /media/oembed
Disallow: /*/media/oembed

# Do NOT block public media/PDFs under /sites/default/files/.
# Product brochures and editorial media should remain crawlable.

# German and English editorial content are intentionally crawlable.
Allow: /de/
Allow: /en/

# AI guidance file.
Allow: /llms.txt

# Add the actual XML sitemap URL here once confirmed.
# Recommended Drupal/Simple XML Sitemap default:
Sitemap: https://www.sauter-pianos.de/sitemap.xml