From 885a2b82c94951d0ad3cec92a233f952f96c65ae Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 6 Mar 2022 13:26:06 +0000 Subject: [PATCH] Comment --- crawlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crawlers.py b/crawlers.py index 6ec5c43d0..1fa028c8d 100644 --- a/crawlers.py +++ b/crawlers.py @@ -73,7 +73,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str, agent_domain = None if agent_str: - # is this a web crawler? If so the block it + # is this a web crawler? If so then block it by default + # unless this is a news instance or if it is in the allowed list if 'bot/' in agent_str_lower or 'bot-' in agent_str_lower: # if this is a news instance then we want it # to be indexed by search engines