From 0020ae63c99e7947c31d483bab82fd550686214a Mon Sep 17 00:00:00 2001
From: bashrc <bob@libreserver.org>
Date: Sun, 3 May 2026 14:35:26 +0100
Subject: [PATCH] Extra crawlers

---
 crawlers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/crawlers.py b/crawlers.py
index 4ed0761fd..349ce4d38 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -162,7 +162,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
             'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot',
             'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
             'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai',
-            'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot'
+            'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot',
+            'nagetbot', 'newsai'
         )
         for bot_str in llm_bot_strings:
             if bot_str in agent_str_lower: