More user agent blocks

2026-08-30 15:04:03 +01:00 · 2026-08-30 15:04:03 +01:00 · 377bd7606b
parent c23b110b54
commit 377bd7606b
1 changed files with 2 additions and 1 deletions
--- a/src/crawlers.py
+++ b/src/crawlers.py
@ -23,7 +23,8 @@ from src.data import save_string
 from src.data import is_a_file

 default_user_agent_blocks = [
-    'fedilist', 'ncsc scan', 'fedifetcher', 'example.com'
+    'fedilist', 'ncsc scan', 'fedifetcher', 'example.com',
+    'python-requests', ' curl/', 'urllib/', 'scrapy.org'
 ]