# robots.txt for WordPress site
# https://liveutifree.com

# Rules for legitimate search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Mobile Smartphone
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: YandexImages
User-agent: facebookexternalhit
User-agent: Naverbot
User-agent: Seznambot
Allow: /

# Default bot rules - placing this higher to define global rules first
User-agent: *

# WP Defender - Begin
Disallow: /0ef508173e053f82
# WP Defender - End

# Allow specific resource files
Allow: /wp-includes/*.css
Allow: /wp-includes/*.js
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Allow: /wp-content/uploads/*.jpg
Allow: /wp-content/uploads/*.jpeg
Allow: /wp-content/uploads/*.png
Allow: /wp-content/uploads/*.gif
Allow: /wp-content/uploads/*.webp
Allow: /favicon.ico

# Default bot rules continued
User-agent: *
Crawl-delay: 20

# Block URL parameters to prevent duplicate content
Disallow: /*?s=*
Disallow: /*?p=*
Disallow: /*&s=*
Disallow: /*&p=*
Disallow: /*?filter=*
Disallow: /*&filter=*
Disallow: /*?orderby=*
Disallow: /*&orderby=*
Disallow: /*?m=*
Disallow: /*&m=*
Disallow: /*?cat=*
Disallow: /*&cat=*
Disallow: /*?lang=*
Disallow: /*&lang=*

# Block WordPress system files and directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/wp-rocket-cache/
Disallow: /wp-content/cache/wp-rocket/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backup-db/
Disallow: /wp-content/uploads/wpcf7_uploads/
Disallow: /wp-content/uploads/cache/
Disallow: /wp-content/uploads/smush-webp/
Disallow: /wp-content/themes/*/readme.txt
Disallow: /wp-content/themes/*/README.txt
Disallow: /wp-content/themes/*/license.txt
Disallow: /wp-content/themes/*/LICENSE.txt
Disallow: /wp-content/advanced-cache.php
Disallow: /wp-content/wp-cache-config.php
Disallow: /wp-content/debug.log
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-mail.php
Disallow: /wp-config.php
Disallow: /wp-config-sample.php
Disallow: /wp-cron.php
Disallow: /wp-signup.php
Disallow: /wp-login.php
Disallow: /wp-activate.php
Disallow: /wp-comments-post.php
Disallow: /wp-trackback.php

# Block sensitive pages
User-agent: *
Disallow: /category/uncategorized/
Disallow: /tag/
Disallow: */trackback/
Disallow: */feed/
Disallow: */embed/
Disallow: */page/
Disallow: */comments/feed/
Disallow: /author/*/
Disallow: /404/
Disallow: /search/
Disallow: */replytocom=*

# WPML and multilingual specific rules
# Allow language-specific paths while maintaining proper crawling
Allow: /*/wp-content/uploads/
Allow: /*/wp-content/themes/

# Handle language paths
# English at root, other languages with ISO codes
Allow: /es/
Allow: /fr/
Allow: /de/
Allow: /it/
Allow: /pt/
Allow: /ru/
Allow: /zh/
Allow: /ja/
Allow: /ar/
Allow: /fr/
Allow: /nl/
# Add any other language codes you plan to use

# Block duplicate WPML content
Disallow: /wp-content/cache/wpml/
Disallow: /wp-content/uploads/cache/wpml/
Disallow: /wp-content/plugins/sitepress-multilingual-cms/

# Limit LLM crawlers and AI data collection bots
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: FacebookBot
User-agent: PerplexityBot
Crawl-delay: 15
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /?s=
Disallow: /*?*

# Limit aggressive SEO bots
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
Crawl-delay: 10

# Block potentially harmful or resource-intensive bots
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: SEOkicks
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: ia_archiver
User-agent: alexibot
User-agent: Barkrowler
User-agent: AspiegelBot
User-agent: Aboundexbot
User-agent: ExtLinksBot
User-agent: spbot
User-agent: twengabot
User-agent: Applebot
User-agent: BLEXBot
User-agent: LinkpadBot
User-agent: magpie-crawler
User-agent: HTTrack
User-agent: wget
User-agent: Nutch
User-agent: WBSearchBot
User-agent: MegaIndex
User-agent: ltx71
User-agent: TheNomad
User-agent: Nimbostratus-Bot
User-agent: DomainStatsBot
Disallow: /

Sitemap: https://liveutifree.com/sitemap.xml