# WordPress robots.txt - Enhanced Version
# Last updated: August 12, 2025

# ====================
# ALLOW MAJOR SEARCH ENGINES
# ====================
User-Agent: Googlebot
Allow: /

User-Agent: Bingbot
Allow: /

User-Agent: DuckDuckBot
Allow: /

# ====================
# BLOCK SEO TOOLS & SCRAPERS
# ====================
User-Agent: AhrefsBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-Agent: DataForSeoBot
Disallow: /

# ====================
# BLOCK AI CRAWLERS
# ====================
User-Agent: GPTBot
Disallow: /

User-Agent: CCBot
Disallow: /

User-Agent: ChatGPT-User
Disallow: /

# User-Agent: ClaudeBot
# Disallow: /

# User-Agent: Anthropic-AI
# Disallow: /

# User-Agent: Claude-Web
# Disallow: /

User-Agent: Cohere-ai
Disallow: /

User-Agent: perplexitybot
Disallow: /

User-Agent: Bytespider
Disallow: /

# ====================
# BLOCK COMMON SCRAPING TOOLS
# ====================
User-Agent: Scrapy
Disallow: /

User-Agent: Python-urllib
Disallow: /

User-Agent: curl
Disallow: /

User-Agent: wget
Disallow: /

# ====================
# BLOCK AGGRESSIVE SEARCH BOTS
# ====================
User-Agent: Yandex
Disallow: /

User-Agent: Baiduspider
Disallow: /

# ====================
# GENERAL RULES FOR ALL OTHER BOTS
# ====================
User-Agent: *
Crawl-delay: 5
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /refer/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /xmlrpc.php
Disallow: /wp-config.php
Disallow: /.htaccess
Disallow: /author/
Disallow: /?author=*
Disallow: /feed/
Disallow: /*/feed/
Disallow: /?s=
Disallow: /*?*s=
Disallow: /*?*p=
Disallow: /*?attachment_id=*
Disallow: /*?utm_source=*
Disallow: /*?utm_medium=*
Disallow: /*?utm_campaign=*
Disallow: /*?utm_term=*
Disallow: /*?utm_content=*
Disallow: /*?replytocom
Disallow: /*.php$
Disallow: /*.cgi$
Allow: /wp-content/uploads/

# ====================
# SITEMAP
# ====================
Sitemap: https://www.thespoken.cc/sitemap_index.xml