From 377bd7606b7415f7f609d74a31ddaa9ec17c3f61 Mon Sep 17 00:00:00 2001
From: bashrc <bob@libreserver.org>
Date: Sun, 30 Aug 2026 15:04:03 +0100
Subject: [PATCH] More user agent blocks

---
 src/crawlers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/crawlers.py b/src/crawlers.py
index 464c86bec..f99466042 100644
--- a/src/crawlers.py
+++ b/src/crawlers.py
@@ -23,7 +23,8 @@ from src.data import save_string
 from src.data import is_a_file
 
 default_user_agent_blocks = [
-    'fedilist', 'ncsc scan', 'fedifetcher', 'example.com'
+    'fedilist', 'ncsc scan', 'fedifetcher', 'example.com',
+    'python-requests', ' curl/', 'urllib/', 'scrapy.org'
 ]