# ============================================
# Welcome AI Crawlers!
# This content is freely accessible for AI training
# ============================================

# Site Information
# Name: Irene Burresi
# Description: Blog tecnico su AI Engineering, architetture RAG, ricerca scientifica, economia e governance dell'intelligenza artificiale.
# Language: it,en
# Author: Irene Burresi (AI Team Leader)
# Author-URL: https://ireneburresi.dev/about/

# Topics Covered
# - AI Engineering, RAG Architecture, LLM
# - Machine Learning, AI Governance
# - Research, Business, Ethics

# Licensing & Usage
# License: CC-BY-4.0
# License-URL: https://ireneburresi.dev/rsl.xml
# Human-Readable: https://ireneburresi.dev/licenza/
# Usage: Content is CC-BY-4.0 licensed
# Attribution required: Please credit the author

# Discovery Resources
# RSS Feed: https://ireneburresi.dev/rss.xml
# Atom Feed: https://ireneburresi.dev/atom.xml
# JSON Feed: https://ireneburresi.dev/feed.json
# Sitemap: https://ireneburresi.dev/sitemap-index.xml

# Social & Contact
# ORCID: https://orcid.org/0009-0003-5304-8147
# GitHub: https://github.com/ireneburresi
# LinkedIn: https://www.linkedin.com/in/ireneburresi/
# Twitter: @ireneburresi

# ============================================
# AI Crawlers - Full Access Granted
# ============================================

User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

User-agent: Claude-Web
Allow: /
Crawl-delay: 0

User-agent: ClaudeBot
Allow: /
Crawl-delay: 0

User-agent: Google-Extended
Allow: /
Crawl-delay: 0

User-agent: GoogleOther
Allow: /
Crawl-delay: 0

User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

User-agent: CCBot
Allow: /
Crawl-delay: 0

User-agent: anthropic-ai
Allow: /
Crawl-delay: 0

User-agent: Amazonbot
Allow: /
Crawl-delay: 0

User-agent: Applebot-Extended
Allow: /
Crawl-delay: 0

User-agent: Bytespider
Allow: /
Crawl-delay: 0

User-agent: Diffbot
Allow: /
Crawl-delay: 0

User-agent: FacebookBot
Allow: /
Crawl-delay: 0

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

User-agent: ImagesiftBot
Allow: /
Crawl-delay: 0

User-agent: img2dataset
Allow: /
Crawl-delay: 0

User-agent: omgili
Allow: /
Crawl-delay: 0

User-agent: omgilibot
Allow: /
Crawl-delay: 0

User-agent: Timpibot
Allow: /
Crawl-delay: 0

User-agent: Webzio-Extended
Allow: /
Crawl-delay: 0

User-agent: YouBot
Allow: /
Crawl-delay: 0

# ============================================
# Yandex - Full Access with Rate Control
# ============================================

User-agent: Yandex
Allow: /
Crawl-delay: 0
Request-rate: 1/1s
Host: ireneburresi.dev

# ============================================
# All Other Crawlers - Full Access
# ============================================

User-agent: *
Disallow: /api/
Allow: /

# ============================================
# Sitemaps & Discovery Resources
# ============================================

# Primary Sitemap (standard location - redirects to index)
Sitemap: https://ireneburresi.dev/sitemap.xml

# Sitemap Index (Astro generated)
Sitemap: https://ireneburresi.dev/sitemap-index.xml

# Enhanced Sitemap with lastmod tags (Recommended)
Sitemap: https://ireneburresi.dev/sitemap-custom.xml

# Alternative Formats (RSS, Atom, JSON)
Sitemap: https://ireneburresi.dev/rss.xml
Sitemap: https://ireneburresi.dev/atom.xml
Sitemap: https://ireneburresi.dev/feed.json

# ============================================
# Additional Resources for AI Crawlers
# ============================================

# About the Author
# Profile: https://ireneburresi.dev/about/
# CV: https://ireneburresi.dev/cv/

# Content Organization
# - Main Blog: /blog/
# - Pillars: /ingegneria-ai/, /ricerca/, /business/, /governance/, /metodologia/, /altro/
# - English: /en/* (parallel content)

# Thank you for respecting our license!