Anti-AI measures

public/robots.txt

@@ -1,315 +1,429 @@
- User-agent: 2^32$
-User-agent: AddSearchBot
-User-agent: AdsBot-Google
-User-agent: Agentic
-User-agent: AhrefsBot
+User-agent: 2^32$
 User-agent: .ai
-User-agent: AI21 Labs
-User-agent: AI2Bot
-User-agent: Ai2Bot-Dolma
-User-agent: AI2Bot-Dolma
+User-agent: -ai
+User-agent: _ai
+User-agent: ai.
+User-agent: ai-
+User-agent: ai_
+User-agent: ai=
+User-agent: AddSearchBot
+User-agent: Agentic
+User-agent: AgentQL
+User-agent: Agent 3
+User-agent: Agent API
+User-agent: Agent-
+User-agent: AI Agent
 User-agent: AI Article Writer
-User-agent: AIBot
+User-agent: AI Assistant
+User-agent: AI Chat
 User-agent: AI Content Detector
+User-agent: AI Detection
 User-agent: AI Dungeon
-User-agent: aiHitBot
-User-agent: AIMatrix
-User-agent: AISearchBot
-User-agent: AI Search Engine
+User-agent: AI Journalist
+User-agent: AI Legion
+User-agent: AI RAG
+User-agent: AI Search
 User-agent: AI SEO Crawler
 User-agent: AI Training
-User-agent: AITraining
+User-agent: AI Web
 User-agent: AI Writer
+User-agent: AI2
+User-agent: AIBot
+User-agent: aiHitBot
+User-agent: AIMatrix
+User-agent: Airesearch
+User-agent: AISearch
+User-agent: AITraining
 User-agent: Alexa
+User-agent: Alice Yandex
+User-agent: AliGenie
+User-agent: AliyunSec
 User-agent: Alpha AI
 User-agent: AlphaAI
-User-agent: a[mazing]{42}(robot)
-User-agent: Amazon Bedrock
-User-agent: Amazonbot
-User-agent: AmazonBot
-User-agent: Amazon Comprehend
-User-agent: Amazon-Kendra
-User-agent: Amazon Lex
-User-agent: Amazon Sagemaker
-User-agent: Amazon Silk
-User-agent: Amazon Textract
+User-agent: Amazon
 User-agent: Amelia
 User-agent: AndersPinkBot
-User-agent: Andibot
+User-agent: AndiBot
+User-agent: Anomura
+User-agent: Anonymous AI
 User-agent: Anthropic
-User-agent: anthropic-ai
 User-agent: AnyPicker
 User-agent: Anyword
 User-agent: Applebot
-User-agent: Applebot-Extended
+User-agent: Aria AI
 User-agent: Aria Browse
 User-agent: Articoolo
+User-agent: Ask AI
+User-agent: Atlassian
+User-agent: AutoGen
 User-agent: Automated Writer
+User-agent: AutoML
+User-agent: Autonomous RAG
+User-agent: AutoRAG
 User-agent: Awario
-User-agent: AwarioBot
-User-agent: AwarioRssBot
-User-agent: AwarioSmartBot
+User-agent: AWS Trainium
 User-agent: Azure
+User-agent: BabyAGI
+User-agent: BabyCatAGI
 User-agent: BardBot
-User-agent: bedrockbot
-User-agent: bigsur.ai
-User-agent: BLEXBot
-User-agent: Brave Leo
-User-agent: Brightbot 1.0
+User-agent: Basic RAG
+User-agent: Bedrock
+User-agent: Big Sur
+User-agent: Bigsur
+User-agent: Botsonic
+User-agent: Brightbot
+User-agent: Browser MCP Agent
+User-agent: Browser Use
+User-agent: BuddyBot
+User-agent: Bytebot
 User-agent: ByteDance
 User-agent: Bytespider
+User-agent: CarynAI
 User-agent: CatBoost
-User-agent: CCBot
 User-agent: CC-Crawler
-User-agent: ChatGLM
-User-agent: ChatGPT Agent
-User-agent: ChatGPT-User
-User-agent: ChatGPT-User/2.0
+User-agent: CCBot
+User-agent: Chai
+User-agent: Channel3Bot
+User-agent: Character
+User-agent: Charstar AI
+User-agent: Chatbot
+User-agent: Chatsonic
+User-agent: ChatUser
 User-agent: Chinchilla
 User-agent: Claude
-User-agent: ClaudeBot
-User-agent: Claude-SearchBot
-User-agent: Claude-User
-User-agent: claude-web
-User-agent: Claude-Web
 User-agent: ClearScope
-User-agent: CloudVertexBot
+User-agent: Clearview
+User-agent: Cognitive AI
 User-agent: Cohere
-User-agent: cohere-ai
-User-agent: cohere-training-data-crawler
 User-agent: Common Crawl
 User-agent: CommonCrawl
-User-agent: ContentAtScale
-User-agent: ContentBot
-User-agent: Contentedge
+User-agent: Compass
 User-agent: Content Harmony
 User-agent: Content King
 User-agent: Content Optimizer
 User-agent: Content Samurai
+User-agent: ContentAtScale
+User-agent: ContentBot
+User-agent: Contentedge
+User-agent: ContentShake
 User-agent: Conversion AI
 User-agent: Copilot
 User-agent: CopyAI
 User-agent: Copymatic
 User-agent: Copyscape
+User-agent: CoreWeave
+User-agent: Corrective RAG
 User-agent: Cotoyogi
-User-agent: crawler.with.dots
+User-agent: CRAB
+User-agent: Crawl4AI
 User-agent: CrawlQ AI
 User-agent: Crawlspace
 User-agent: Crew AI
 User-agent: CrewAI
-User-agent: curl|sudo bash
+User-agent: Crushon AI
+User-agent: Cypex
 User-agent: DALL-E
-User-agent: DataForSeoBot
+User-agent: DarkBard
+User-agent: DarkBERT
+User-agent: DataFor
 User-agent: DataProvider
 User-agent: Datenbank Crawler
 User-agent: DeepAI
+User-agent: Deep AI
 User-agent: DeepL
 User-agent: DeepMind
+User-agent: Deep Research
+User-agent: DeepResearch
 User-agent: DeepSeek
 User-agent: Devin
-User-agent: diffbot
 User-agent: Diffbot
+User-agent: Dolma
 User-agent: Doubao AI
 User-agent: DuckAssistBot
-User-agent: Echobot Bot
-User-agent: EchoboxBot
-User-agent: Facebookbot
+User-agent: DuckDuckGo Chat
+User-agent: DuckDuckGo-Enhanced
+User-agent: Echobot
+User-agent: Echobox
+User-agent: Elixir
 User-agent: FacebookBot
-User-agent: facebookexternalhit
 User-agent: FacebookExternalHit
-User-agent: Factset_spyderbot
+User-agent: Factset
 User-agent: Falcon
+User-agent: FIRE-1
+User-agent: Firebase
 User-agent: Firecrawl
-User-agent: FirecrawlAgent
 User-agent: Flyriver
 User-agent: Frase AI
 User-agent: FriendlyCrawler
-User-agent: Fuzz Faster U Fool
-User-agent: Fuzz Faster U Fool v2.0.0
+User-agent: Gato
 User-agent: Gemini
-User-agent: Gemini-Deep-Research
 User-agent: Gemma
+User-agent: Gen AI
 User-agent: GenAI
+User-agent: Generative
 User-agent: Genspark
-User-agent: Gigabot
+User-agent: Gentoo-chat
+User-agent: Ghostwriter
+User-agent: GigaChat
 User-agent: GLM
-User-agent: GoogleAgent-Mariner
-User-agent: Google-CloudVertexBot
-User-agent: Google-Extended
-User-agent: Google-Firebase
-User-agent: GoogleOther
-User-agent: GoogleOther-Image
-User-agent: GoogleOther-Video
+User-agent: GodMode
 User-agent: Goose
 User-agent: GPT
-User-agent: GPTBot
 User-agent: Grammarly
 User-agent: Grendizer
 User-agent: Grok
+User-agent: Groq
 User-agent: GT Bot
 User-agent: GTBot
+User-agent: GTP
 User-agent: Hemingway Editor
-User-agent: Hugging Face
+User-agent: Hetzner
+User-agent: Hugging
+User-agent: Hunyuan
+User-agent: Hybrid Search RAG
 User-agent: Hypotenuse AI
-User-agent: iaskspider
-User-agent: iaskspider/2.0
+User-agent: iAsk
+User-agent: IbouBot
 User-agent: ICC-Crawler
+User-agent: Ideogram
 User-agent: ImageGen
+User-agent: Imagen
 User-agent: ImagesiftBot
+User-agent: ImageSpider
 User-agent: img2dataset
 User-agent: imgproxy
-User-agent: Inferkit
 User-agent: INK Editor
 User-agent: INKforall
+User-agent: Instructor
 User-agent: IntelliSeek
+User-agent: Inferkit
 User-agent: ISSCyberRiskCrawler
-User-agent: Is this a crawler?
-User-agent: JasperAI
+User-agent: Janitor AI
+User-agent: Jasper
+User-agent: Jenni AI
+User-agent: Julius AI
 User-agent: Kafkai
+User-agent: Kaggle
 User-agent: Kangaroo
-User-agent: Kangaroo Bot
 User-agent: Keyword Density AI
+User-agent: Kimi
+User-agent: Klaviyo
 User-agent: Knowledge
 User-agent: KomoBot
+User-agent: Kruti
+User-agent: KunatoCrawler
+User-agent: LAIONDownloader
+User-agent: LangChain
+User-agent: Langfuse
+User-agent: Language AI
+User-agent: Le Chat
+User-agent: Lensa
+User-agent: Leonardo
+User-agent: Lightpanda
+User-agent: LightRAG
 User-agent: LinerBot
-User-agent: LinkedInBot
+User-agent: LinkupBot
 User-agent: LLaMA
-User-agent: LLMs
+User-agent: LCC
+User-agent: LLM
+User-agent: Local RAG Agent
+User-agent: Lovable
+User-agent: Magistral
 User-agent: magpie-crawler
+User-agent: MAI-Image
+User-agent: Manus
 User-agent: MarketMuse
+User-agent: MBZUAI
 User-agent: Meltwater
-User-agent: Meta AI
 User-agent: Meta-AI
-User-agent: MetaAI
 User-agent: Meta-External
-User-agent: meta-externalagent
-User-agent: Meta-ExternalAgent
-User-agent: meta-externalfetcher
-User-agent: Meta-ExternalFetcher
+User-agent: Meta-Webindexer
+User-agent: Meta AI
+User-agent: MetaAI
 User-agent: MetaTagBot
-User-agent: meta-webindexer
+User-agent: Middleware
+User-agent: Midjourney
+User-agent: Mini AGI
+User-agent: MiniMax
+User-agent: Mintlify
 User-agent: Mistral
-User-agent: MistralAI-User
-User-agent: MistralAI-User/1.0
-User-agent: MJ12bot
-User-agent: MyCentralAIScraperBot
+User-agent: Mixtral
+User-agent: model-training
+User-agent: Molmo
+User-agent: Monica
+User-agent: Nano
 User-agent: Narrative
 User-agent: NeevaBot
-User-agent: netEstate Imprint Crawler
-User-agent: NeuralSEO
+User-agent: netEstate
 User-agent: Neural Text
+User-agent: NeuralSEO
+User-agent: NinjaAI
+User-agent: NodeZero
+User-agent: NotebookLM
 User-agent: Nova Act
 User-agent: NovaAct
-User-agent: Nutch
+User-agent: Nytheon
 User-agent: OAI-SearchBot
-User-agent: omgili
+User-agent: OAI SearchBot
+User-agent: OASIS
+User-agent: Olivia
+User-agent: OLM
 User-agent: Omgili
-User-agent: omgilibot
-User-agent: Omgilibot
-User-agent: OmniExplorer_Bot
+User-agent: Onyx
 User-agent: Open AI
+User-agent: Open Interpreter
+User-agent: OpenAGI
 User-agent: OpenAI
 User-agent: OpenBot
+User-agent: OpenInterpreter
+User-agent: OpenPi
+User-agent: OpenRouter
 User-agent: OpenText AI
 User-agent: Operator
 User-agent: Outwrite
+User-agent: OVHcloud
+User-agent: Owlin
 User-agent: Page Analyzer AI
+User-agent: PaLM 2
 User-agent: PanguBot
 User-agent: Panscient
-User-agent: panscient.com
 User-agent: Paperlibot
 User-agent: Paraphraser.io
 User-agent: peer39_crawler
+User-agent: Perflexity
 User-agent: Perplexity
-User-agent: PerplexityBot
-User-agent: Perplexity-User
-User-agent: Petalbot
-User-agent: PetalBot
-User-agent: Phindbot
-User-agent: PhindBot
+User-agent: Petal
+User-agent: Phind
 User-agent: PiplBot
-User-agent: Poseidon Research Crawler
-User-agent: prefetch-proxy
+User-agent: Pixmo
+User-agent: Playwright
+User-agent: PoeBot
+User-agent: PoeSearchBot
+User-agent: Poggio
+User-agent: Poseidon
 User-agent: ProWritingAid
-User-agent: psbot
-User-agent: python-requests
-User-agent: QualifiedBot
+User-agent: Proximic
+User-agent: PulsarRPA
+User-agent: Puppeteer
+User-agent: Python AI
+User-agent: Python lxml
+User-agent: PyTorch
+User-agent: Qualified
+User-agent: Quark
 User-agent: QuillBot
-User-agent: quillbot.com
+User-agent: Qopywriter
+User-agent: Qwen
+User-agent: RAG Agent
+User-agent: RAG Azure AI
+User-agent: RAG Chatbot
+User-agent: RAG Database
+User-agent: RAG IS
+User-agent: RAG Pipeline
+User-agent: RAG Search
+User-agent: RAG with
+User-agent: RAG-
+User-agent: RAG_
+User-agent: Raptor
+User-agent: React Agent
+User-agent: Reasoning
+User-agent: Redis AI RAG
+User-agent: Reka Flash
+User-agent: Replicate
 User-agent: RobotSpider
-User-agent: Robozilla
+User-agent: RunPod
 User-agent: Rytr
 User-agent: SaplingAI
 User-agent: SBIntuitionsBot
+User-agent: Scala
 User-agent: Scalenut
-User-agent: Scraper
-User-agent: Scrapy
+User-agent: Scrap
 User-agent: ScriptBook
 User-agent: Seekr
-User-agent: SemrushBot-OCOB
-User-agent: SemrushBot-SWA
-User-agent: sentibot
-User-agent: Sentibot
-User-agent: SentiBot
+User-agent: SEObot
 User-agent: SEO Content Machine
 User-agent: SEO Robot
+User-agent: SemrushBot
+User-agent: Sentibot
+User-agent: Serper
 User-agent: ShapBot
 User-agent: Sidetrade
-User-agent: Sidetrade indexer bot
 User-agent: Simplified AI
 User-agent: Sitefinity
 User-agent: Skydancer
+User-agent: Skyvern
 User-agent: SlickWrite
+User-agent: SmartBot
 User-agent: Sonic
+User-agent: Sora
+User-agent: Spider/2
+User-agent: SpiderCreator
+User-agent: Spin Rewrite
 User-agent: Spinbot
-User-agent: Spin Rewriter
 User-agent: Stability
+User-agent: Stable Diffusion
 User-agent: StableDiffusionBot
-User-agent: star***crawler
+User-agent: Stagehand
+User-agent: Stealth
 User-agent: Sudowrite
 User-agent: SummalyBot
 User-agent: Super Agent
+User-agent: Superagent
+User-agent: SuperAGI
 User-agent: Surfer AI
-User-agent: Teoma
+User-agent: Taiko
+User-agent: TavilyBot
 User-agent: TerraCotta
 User-agent: Text Blaze
 User-agent: TextCortex
-User-agent: The Knowledge AI
 User-agent: Thinkbot
-User-agent: ThinkChaos
+User-agent: Thordata
 User-agent: TikTokSpider
 User-agent: Timpibot
-User-agent: TimpiBot
+User-agent: Tinybird
+User-agent: Together
+User-agent: Traefik
+User-agent: Tulu
+User-agent: Tülu
 User-agent: TurnitinBot
+User-agent: TwinAgent
+User-agent: uAgents
+User-agent: UI-TARS
 User-agent: VelenPublicWebCrawler
+User-agent: Venus Chub AI
 User-agent: Vidnami AI
+User-agent: Vision RAG
 User-agent: WARDBot
+User-agent: WebSurfer
+User-agent: WebText
+User-agent: WebVoyager
 User-agent: Webzio
-User-agent: webzio-extended
-User-agent: Webzio-Extended
+User-agent: WeChat
 User-agent: Whisper
 User-agent: WordAI
 User-agent: Wordtune
-User-agent: WormsGTP
-User-agent: wpbot
 User-agent: WPBot
 User-agent: Writecream
 User-agent: WriterZen
 User-agent: Writescope
 User-agent: Writesonic
+User-agent: WRTNBot
 User-agent: xAI
+User-agent: Xanthorox
 User-agent: xBot
+User-agent: XLNet
 User-agent: YaK
+User-agent: YaML
 User-agent: YandexAdditional
-User-agent: YandexAdditionalBot
-User-agent: Youbot
 User-agent: YouBot
-User-agent: Zerochat
-User-agent: Zero GTP
+User-agent: Zanista
+User-agent: Zendesk
+User-agent: Zero
 User-agent: Zhipu
+User-agent: Zhuque AI
 User-agent: Zimm
 Disallow: /
 Disallow: *
 DisallowAITraining: /
 
 Content-Usage: ai=n
+# Blocklist via: https://perishablepress.com/ultimate-ai-block-list/