91 lines
1.2 KiB
Plaintext
91 lines
1.2 KiB
Plaintext
# robots.txt – KI-Crawler blockiert
|
||
|
||
# OpenAI / ChatGPT
|
||
User-agent: GPTBot
|
||
Disallow: /
|
||
|
||
User-agent: ChatGPT-User
|
||
Disallow: /
|
||
|
||
User-agent: OAI-SearchBot
|
||
Disallow: /
|
||
|
||
# Anthropic / Claude
|
||
User-agent: ClaudeBot
|
||
Disallow: /
|
||
|
||
User-agent: Claude-Web
|
||
Disallow: /
|
||
|
||
User-agent: anthropic-ai
|
||
Disallow: /
|
||
|
||
# Google AI (Gemini / Bard)
|
||
User-agent: Google-Extended
|
||
Disallow: /
|
||
|
||
# Meta AI
|
||
User-agent: FacebookBot
|
||
Disallow: /
|
||
|
||
User-agent: Meta-ExternalAgent
|
||
Disallow: /
|
||
|
||
User-agent: Meta-ExternalFetcher
|
||
Disallow: /
|
||
|
||
# Apple
|
||
User-agent: Applebot-Extended
|
||
Disallow: /
|
||
|
||
# Amazon Alexa
|
||
User-agent: Amazonbot
|
||
Disallow: /
|
||
|
||
# Perplexity AI
|
||
User-agent: PerplexityBot
|
||
Disallow: /
|
||
|
||
# Cohere
|
||
User-agent: cohere-ai
|
||
Disallow: /
|
||
|
||
# Common Crawl (Trainingsdaten für viele KI-Modelle)
|
||
User-agent: CCBot
|
||
Disallow: /
|
||
|
||
# Bytedance / TikTok
|
||
User-agent: Bytespider
|
||
Disallow: /
|
||
|
||
# DiffBot
|
||
User-agent: Diffbot
|
||
Disallow: /
|
||
|
||
# ImagesiftBot
|
||
User-agent: ImagesiftBot
|
||
Disallow: /
|
||
|
||
# Omgili / Webz.io
|
||
User-agent: omgili
|
||
Disallow: /
|
||
|
||
User-agent: omgilibot
|
||
Disallow: /
|
||
|
||
# Webz.io
|
||
User-agent: webzio-extended
|
||
Disallow: /
|
||
|
||
# Timpibot
|
||
User-agent: Timpibot
|
||
Disallow: /
|
||
|
||
# YouBot (You.com)
|
||
User-agent: YouBot
|
||
Disallow: /
|
||
|
||
# Scrapy (allgemeiner Scraper)
|
||
User-agent: Scrapy
|
||
Disallow: /
|