From 259d51d00f6e2aa0ef85ac26e8f5067c7ddb5b22 Mon Sep 17 00:00:00 2001
From: bashrc <bob@libreserver.org>
Date: Tue, 14 Apr 2026 20:20:27 +0100
Subject: [PATCH] Extra llm crawlers

---
 crawlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crawlers.py b/crawlers.py
index 212063d34..0f4b78856 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -167,7 +167,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
             'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot',
             'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
             'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai',
-            'kagi-fetcher'
+            'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot'
         )
         for bot_str in llm_bot_strings:
             if bot_str in agent_str_lower: