From 2234167582ac02f289ca5abf2026ea1f727e2bd7 Mon Sep 17 00:00:00 2001 From: bashrc Date: Fri, 9 Oct 2026 18:19:58 +0100 Subject: [PATCH] Extra llm crawlers --- src/crawlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/crawlers.py b/src/crawlers.py index 22e98d89f..7dd6cf1cc 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -167,7 +167,11 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae', 'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher', 'kimi-user', 'querit-searchbot', 'useai', 'cursor', - 'reflectionbot', 'exasearchbot', 'lightpanda', 'ilands.ai' + 'reflectionbot', 'exasearchbot', 'lightpanda', 'ilands.ai', + 'keenablebot', 'agentdatabot', 'bixelbot', + 'cloudflarebrowserrenderingcrawler', 'kimi-agent', 'qodercli', + 'doubaobot', 'erniebot', 'qwenbot', 'kimi-searchbot', + 'kimibot', 'mistralai-training', 'oai-adsbot', 'diffbot-user' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: