From 5f358a1005d23a6db6e6a359e3eeb60e8c7ab1d7 Mon Sep 17 00:00:00 2001 From: bashrc Date: Mon, 29 Jun 2026 10:59:14 +0100 Subject: [PATCH] More LLM crawlers --- src/crawlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/crawlers.py b/src/crawlers.py index 387a1a860..9146d4595 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -163,7 +163,9 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta', 'terra cotta', 'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', - 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae' + 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae', + 'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher', + 'kimi-user', 'querit-searchbot', 'useai', 'cursor' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: