# ============================================================
# robots.txt - mesaitalia.it
# Optimized for SEO and AI Crawlers
# Updated: 2026-01-20
# ============================================================

# === GENERAL CRAWLERS ===
User-agent: *
Allow: /
Crawl-delay: 1

# WordPress optimization
Allow: /wp-content/uploads/
Allow: /wp-content/cache/
Allow: /wp-admin/admin-ajax.php

# Security - block admin and sensitive areas
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /*?s=*
Disallow: /author/
Disallow: /tag/
Disallow: */feed/
Disallow: */trackback/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /wp-config.php
Disallow: /.git/
Disallow: /.env

# === SEO CRAWLERS - FULL ACCESS ===
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

# === AI CRAWLERS - WELCOME & FULL ACCESS ===
# OpenAI GPTBot - for ChatGPT training and browsing
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic ClaudeBot - for Claude AI
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Google AI (Bard/Gemini) extended crawler
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Common Crawl - used by many AI systems
User-agent: CCBot
Allow: /

# Apple AI (Applebot for Siri/Spotlight)
User-agent: Applebot
Allow: /

# === BLOCK AGGRESSIVE SCRAPERS ===
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# === SITEMAPS ===
Sitemap: https://mesaitalia.it/sitemap_index.xml
Sitemap: https://mesaitalia.it/post-sitemap.xml
Sitemap: https://mesaitalia.it/page-sitemap.xml
Sitemap: https://mesaitalia.it/product-sitemap.xml

# AI-Optimized Sitemap
Sitemap: https://mesaitalia.it/sitemap-ai.xml