diff --git a/src/crawlers.py b/src/crawlers.py index 1c66cbfc1..464c86bec 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -165,7 +165,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae', 'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher', - 'kimi-user', 'querit-searchbot', 'useai', 'cursor' + 'kimi-user', 'querit-searchbot', 'useai', 'cursor', + 'reflectionbot', 'exasearchbot', 'lightpanda' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: