diff --git a/crawlers.py b/crawlers.py index 212063d34..0f4b78856 100644 --- a/crawlers.py +++ b/crawlers.py @@ -167,7 +167,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot', 'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta', 'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai', - 'kagi-fetcher' + 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: