Extra LLM crawler

2024-09-01 20:42:19 +01:00 · 2024-09-01 20:42:19 +01:00 · 0ec43c6323
parent a738e1316e
commit 0ec43c6323
1 changed files with 1 additions and 1 deletions
--- a/crawlers.py
+++ b/crawlers.py
@ -135,7 +135,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
            'facebookbot', 'google-extended', 'piplbot', 'oai-search',
            'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot',
            'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
-            'petalbot'
+            'petalbot', 'ai2bot', 'allenai'
        )
        for bot_str in llm_bot_strings:
            if bot_str in agent_str_lower: