Sitemap: https://www.stripweb.be/sitemap_index_stripweb.xml

# Set crawl delay to 10 seconds for all crawlers
User-agent: *
Crawl-delay: 2

# === Op vraag van cnip (9 feb 2026) ===

# Blokkeer oude .asp URL-paden (crawl trap)
Disallow: /*.asp
Disallow: /*.asp/

# Blokkeer interne zoekresultaten
Disallow: /*/zoeken
Disallow: /*/zoeken?*
Disallow: /*/recherche
Disallow: /*/recherche?*

# --- AI ZOEK-BOTS (sturen verkeer naar de site) ---

# OpenAI / ChatGPT
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: claude-web
Allow: /

# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /