From 6a0c77cbca2f35f4e87e05c345c84f2faafb3582 Mon Sep 17 00:00:00 2001 From: bashrc Date: Sun, 30 Aug 2026 14:51:13 +0100 Subject: [PATCH] Another blocked crawler --- src/crawlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlers.py b/src/crawlers.py index a73faa629..1c66cbfc1 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -23,7 +23,7 @@ from src.data import save_string from src.data import is_a_file default_user_agent_blocks = [ - 'fedilist', 'ncsc scan', 'fedifetcher' + 'fedilist', 'ncsc scan', 'fedifetcher', 'example.com' ]