diff --git a/Files/robots.txt b/Files/robots.txt new file mode 100644 index 0000000..eed9b7c --- /dev/null +++ b/Files/robots.txt @@ -0,0 +1,90 @@ +# robots.txt – KI-Crawler blockiert + +# OpenAI / ChatGPT +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: OAI-SearchBot +Disallow: / + +# Anthropic / Claude +User-agent: ClaudeBot +Disallow: / + +User-agent: Claude-Web +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +# Google AI (Gemini / Bard) +User-agent: Google-Extended +Disallow: / + +# Meta AI +User-agent: FacebookBot +Disallow: / + +User-agent: Meta-ExternalAgent +Disallow: / + +User-agent: Meta-ExternalFetcher +Disallow: / + +# Apple +User-agent: Applebot-Extended +Disallow: / + +# Amazon Alexa +User-agent: Amazonbot +Disallow: / + +# Perplexity AI +User-agent: PerplexityBot +Disallow: / + +# Cohere +User-agent: cohere-ai +Disallow: / + +# Common Crawl (Trainingsdaten für viele KI-Modelle) +User-agent: CCBot +Disallow: / + +# Bytedance / TikTok +User-agent: Bytespider +Disallow: / + +# DiffBot +User-agent: Diffbot +Disallow: / + +# ImagesiftBot +User-agent: ImagesiftBot +Disallow: / + +# Omgili / Webz.io +User-agent: omgili +Disallow: / + +User-agent: omgilibot +Disallow: / + +# Webz.io +User-agent: webzio-extended +Disallow: / + +# Timpibot +User-agent: Timpibot +Disallow: / + +# YouBot (You.com) +User-agent: YouBot +Disallow: / + +# Scrapy (allgemeiner Scraper) +User-agent: Scrapy +Disallow: /