Extra check for search bots

crawlers.py

@@ -126,6 +126,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
         contains_bot_string = False
         for bot_str in bot_strings:
             if bot_str in agent_str_lower:
+                if '://bot' not in agent_str_lower and \
+                   '://robot' not in agent_str_lower:
                     contains_bot_string = True
                     break
         if contains_bot_string: