diff --git a/src/crawlers.py b/src/crawlers.py index 387a1a860..9146d4595 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -163,7 +163,9 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta', 'terra cotta', 'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', - 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae' + 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae', + 'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher', + 'kimi-user', 'querit-searchbot', 'useai', 'cursor' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: