#---Google---
#GooglebotignoriertCrawl-delay(SteuerungueberSearchConsole)
User-agent:Googlebot
Disallow:/kuferweb/
Disallow:/typo3/
Disallow:/typo3conf/

#GoogleAI-Training(Geminietc.)—reinesrobots.txt-Token
User-agent:Google-Extended
Disallow:/kuferweb/

#GooglesonstigeCrawler(AI/Forschung)
User-agent:GoogleOther
Crawl-delay:10
Disallow:/kuferweb/

#GoogleCloud/VertexAI
User-agent:Google-CloudVertexBot
Crawl-delay:10
Disallow:/kuferweb/

#---Bing(+MicrosoftCopilot)---
User-agent:Bingbot
Crawl-delay:3
Disallow:/kuferweb/
Disallow:/typo3/
Disallow:/typo3conf/

#---AI-Bots(willkommen,Crawl-delay10s)---
User-agent:GPTBot
User-agent:ChatGPT-User
User-agent:OAI-SearchBot
User-agent:ClaudeBot
User-agent:anthropic-ai
User-agent:PerplexityBot
User-agent:DeepSeekBot
User-agent:DuckAssistBot
User-agent:Applebot
User-agent:Applebot-Extended
User-agent:Amazonbot
User-agent:Amzn-SearchBot
User-agent:meta-externalagent
User-agent:meta-externalfetcher
User-agent:meta-webindexer
User-agent:YouBot
User-agent:cohere-ai
Crawl-delay:10
Disallow:/kuferweb/

#---SEO-Bots(Crawl-delay5s)---
User-agent:SemrushBot
User-agent:AhrefsBot
User-agent:MJ12bot
User-agent:DotBot
User-agent:DataForSeoBot
User-agent:SEOkicks-Robot
User-agent:ZoominfoBot
User-agent:SERankingBacklinks
User-agent:serpstatbot
User-agent:BLEXBot
User-agent:Rogerbot
User-agent:Sistrix
Crawl-delay:5
Disallow:/kuferweb/

#---Scraper/Spam/irrelevant(komplettgesperrt)---
User-agent:PetalBot
User-agent:Barkrowler
User-agent:Bytespider
User-agent:TikTokSpider
User-agent:YandexBot
User-agent:AwarioBot
User-agent:CensysInspect
User-agent:Scrapy
User-agent:CCBot
User-agent:Ai2Bot
User-agent:Diffbot
User-agent:Baiduspider
User-agent:Sogou
User-agent:360Spider
User-agent:BomboraBot
User-agent:Turnitin
User-agent:PanguBot
User-agent:ChatGLM-Spider
User-agent:Brightbot
User-agent:FirecrawlAgent
User-agent:ApifyBot
User-agent:omgilibot
User-agent:PhindBot
User-agent:TavilyBot
Disallow:/

#---StandardfueralleunbekanntenBots---
User-agent:*
Crawl-delay:2
Disallow:/kuferweb/
Disallow:/typo3/
Disallow:/typo3conf/
Sitemap: https://vhs-trier.de/sitemap.xml