User-agent: *
Allow: /
Allow: /wp-admin/admin-ajax.php

# Block sensitive WordPress Core directories & system files
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/   # Stops plugins from leaking script logic
Disallow: /wp-content/themes/    # Stops themes from leaking core styling logic
Disallow: /wp-json/              # Optional: Block WP REST API if you don't use external apps
Disallow: /readme.html           # Security: Hides WordPress core version details
Disallow: /license.txt           # Security: Standard text file mitigation

# Block common WordPress internal clutter & duplicate content
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-trackback/
Disallow: /xmlrpc.php
Disallow: /cgi-bin/

# Block internal dynamic URLs & parameter sprawl (e.g., query loops, preview states)
Disallow: /?s=                  # Internal WordPress search results
Disallow: /*?*                  # URL parameters (tracking tags, filters - adjust if using ecommerce)
Disallow: /*.php$               # Disallow loose PHP scripts from being directly crawled

# --------------------------------------------------------------------------
# AI SEARCH / RETRIEVAL (RFC 9309 — explicit User-agent blocks required)
# --------------------------------------------------------------------------
# Allow indexing for LLM search citations; same path rules as User-agent: *.

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: PerplexityBot
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-trackback/
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /*?*
Disallow: /*.php$

# --------------------------------------------------------------------------
# AI TRAINING & AGGRESSIVE SCRAPERS
# --------------------------------------------------------------------------
# Block model training and bulk dataset crawlers (does not affect Googlebot Search).

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: Bytespider
Disallow: /

# --------------------------------------------------------------------------
# SITEMAP INDEX
# --------------------------------------------------------------------------
Sitemap: https://swissroc.ch/sitemap_index.xml