# =============================================
# BilTAY Teknoloji - robots.txt
# https://www.biltay.com.tr
# =============================================

# --- Tüm Arama Motoru Botları ---
User-agent: *
# Content Signals (contentsignals.org / draft-romm-aipref-contentsignals):
#   search=yes   → Arama motoru indekslemesi serbest
#   ai-input=yes → Kullanıcı sorularına yanıt üretimi için kaynak olarak kullanılabilir
#   ai-train=no  → Model eğitimi için veri seti olarak kullanılamaz
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Sistem ve API yollarını engelle
Disallow: /api/
Disallow: /_astro/
Disallow: /_image

# Eski / arşiv sayfaları (noindex ile yönetilir, Disallow gerekmez)

# --- AI Crawler Botları ---
# GPTBot (OpenAI)
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# ClaudeBot (Anthropic)
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# Google-Extended (Gemini AI)
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# PerplexityBot
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# CCBot (Common Crawl)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# --- Sitemap ---
Sitemap: https://www.biltay.com.tr/sitemap-index.xml

# --- AI Dokümantasyon ---
# LLMs.txt: https://www.biltay.com.tr/llms.txt

# --- Ajan Keşfi (Agent Discovery) ---
# API Catalog:    https://www.biltay.com.tr/.well-known/api-catalog
# Agent Skills:   https://www.biltay.com.tr/.well-known/agent-skills/index.json
# MCP Server:     https://www.biltay.com.tr/.well-known/mcp/server-card.json