# Puoluetesti / Partitestet (puoluetesti.fi)
#
# Text and data mining reservation (Article 4(3) of Directive (EU) 2019/790, as
# transposed into the Finnish Copyright Act 404/1961): see /kayttoehdot#ia
# (Swedish: /kayttoehdot?lang=sv#ia; English: /kayttoehdot?lang=en#ia).
#
# Policy: we block ONLY the crawlers that their operator says are used to train
# AI models. Search engines (Googlebot, Bingbot...), Google-Extended (it also
# governs Gemini grounding with Google Search) and the search and user-agent
# crawlers that cite their source (OAI-SearchBot, ChatGPT-User, Claude-SearchBot,
# Claude-User, PerplexityBot, Perplexity-User, Meta-WebIndexer,
# Meta-ExternalFetcher...) stay open. No global TDM reservation: it would also
# exclude the services that cite us.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /storage/
Disallow: /lang/

# OpenAI: training (not OAI-SearchBot or ChatGPT-User)
User-agent: GPTBot
Disallow: /

# Anthropic: training (not Claude-SearchBot or Claude-User)
User-agent: ClaudeBot
Disallow: /

# Common Crawl: archive used to train models
User-agent: CCBot
Disallow: /

# Apple: only use for model training (Applebot, search, stays open)
User-agent: Applebot-Extended
Disallow: /

# Meta: model training (not Meta-WebIndexer or Meta-ExternalFetcher)
User-agent: Meta-ExternalAgent
Disallow: /

# ByteDance: training (common name; ByteDance publishes no documentation)
User-agent: Bytespider
Disallow: /

Sitemap: https://puoluetesti.fi/sitemap.xml