diff --git a/src/crawlers.py b/src/crawlers.py index 464c86bec..f99466042 100644 --- a/src/crawlers.py +++ b/src/crawlers.py @@ -23,7 +23,8 @@ from src.data import save_string from src.data import is_a_file default_user_agent_blocks = [ - 'fedilist', 'ncsc scan', 'fedifetcher', 'example.com' + 'fedilist', 'ncsc scan', 'fedifetcher', 'example.com', + 'python-requests', ' curl/', 'urllib/', 'scrapy.org' ]