From 0ec43c63233a5a68f57ced04930f0ae01a739494 Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@libreserver.org>
Date: Sun, 1 Sep 2024 20:42:19 +0100
Subject: [PATCH] Extra LLM crawler

---
 crawlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crawlers.py b/crawlers.py
index 21a3ea214..a94370d8e 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -135,7 +135,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
             'facebookbot', 'google-extended', 'piplbot', 'oai-search',
             'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot',
             'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
-            'petalbot'
+            'petalbot', 'ai2bot', 'allenai'
         )
         for bot_str in llm_bot_strings:
             if bot_str in agent_str_lower: