# robots.txt for arieldistefano.com
# Last updated: 2026-04-17
# RFC 9309 compliant
# Content Signals: https://contentsignals.org/

# Allow all crawlers
User-agent: *
Allow: /

# Content Signals - AI Usage Preferences
# ai-train: Whether content can be used to train AI models
# search: Whether content can be indexed for search
# ai-input: Whether content can be used as input/context for AI responses
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Allow AI crawlers (per RFC 9309 and AI Crawl Control best practices)
User-agent: GPTBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: ChatGPT-User
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: Google-Extended
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: anthropic-ai
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: Claude-Web
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: PerplexityBot
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Specific disallow rules (if needed in the future)
# Disallow: /admin/
# Disallow: /private/
# Disallow: /tmp/

# Sitemap reference
Sitemap: https://arieldistefano.com/sitemap.xml

# Crawl-delay (optional, uncomment if needed)
# Crawl-delay: 10