# robots.txt for openmedscience.com
# Purpose: Allow search engines; guide/limit AI/LLM crawlers; block Scrapy & unwanted paths
# Last updated: 13 Sep 2025

# === Global rules for all crawlers ===
User-agent: *
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /private/
Allow: /wp-admin/admin-ajax.php

# === Block Scrapy framework crawlers ===
User-agent: Scrapy
Disallow: /

# === Search engine bots (explicitly allowed) ===
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

# === AI / LLM crawlers ===
# To block completely, replace "/private/" with "/"

User-agent: Google-Extended
Disallow: /private/

User-agent: GPTBot
Disallow: /private/

User-agent: CCBot
Disallow: /private/

User-agent: ClaudeBot
Disallow: /private/

User-agent: PerplexityBot
Disallow: /private/

# === Sitemaps ===
Sitemap: https://openmedscience.com/sitemap_index.xml
Sitemap: https://openmedscience.com/post-sitemap.xml
Sitemap: https://openmedscience.com/page-sitemap.xml
Sitemap: https://openmedscience.com/category-sitemap.xml
Sitemap: https://openmedscience.com/post_tag-sitemap.xml