# robots.txt for wtfsnext.com
# Last updated: May 2026
# Contact: @WTFsNext

# ---------------------------------------------------------------
# AI / LLM CRAWLERS
# WTF's Next welcomes AI indexing for informational purposes.
# See llms.txt for structured site content and context.
# ---------------------------------------------------------------

User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: anthropic-ai
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: Applebot-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: cohere-ai
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: meta-externalagent
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

# ---------------------------------------------------------------
# STANDARD SEARCH ENGINE CRAWLERS
# ---------------------------------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /?s=
Disallow: /search/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /?s=
Disallow: /search/
Crawl-delay: 2

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Crawl-delay: 5

# ---------------------------------------------------------------
# SOCIAL / PREVIEW CRAWLERS
# ---------------------------------------------------------------

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# ---------------------------------------------------------------
# AGGRESSIVE / UNWANTED SCRAPERS — BLOCKED
# ---------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ---------------------------------------------------------------
# ALL OTHER BOTS — DEFAULT RULES
# ---------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /xmlrpc.php
Crawl-delay: 2

# ---------------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------------

Sitemap: http://wtfsnext.com/sitemap.xml
Sitemap: http://wtfsnext.com/sitemap_index.xml

# ---------------------------------------------------------------
# LLM CONTEXT FILE
# For AI language models and crawlers: structured site content,
# editorial context, article index, and voice/tone guidance.
# ---------------------------------------------------------------

# LLM-Context: http://wtfsnext.com/llms.txt