diff --git a/crawlers.py b/crawlers.py index 9c5a5cd73..a587cd092 100644 --- a/crawlers.py +++ b/crawlers.py @@ -139,7 +139,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler', 'googleother', 'icc-crawler', 'scrapy', 'timpibot', 'velenpublic', 'webzio-extended', 'cohere-ai', 'facebookexternal', - 'img2dataset' + 'img2dataset', 'isscyberriskcrawler', 'sidetrade', 'kangaroo', + 'iaskspider' ) for bot_str in llm_bot_strings: if bot_str in agent_str_lower: