diff --git a/crawlers.py b/crawlers.py
index 6ec5c43d0..1fa028c8d 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -73,7 +73,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
     agent_domain = None
 
     if agent_str:
-        # is this a web crawler? If so the block it
+        # is this a web crawler? If so then block it by default
+        # unless this is a news instance or if it is in the allowed list
         if 'bot/' in agent_str_lower or 'bot-' in agent_str_lower:
             # if this is a news instance then we want it
             # to be indexed by search engines