More robot strings

crawlers.py

@@ -122,7 +122,13 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
     if agent_str:
         # is this a web crawler? If so then block it by default
         # unless this is a news instance or if it is in the allowed list
-        if 'bot/' in agent_str_lower or 'bot-' in agent_str_lower:
+        bot_strings = ('bot/', 'bot-', '/bot', '/robot')
+        contains_bot_string = False
+        for bot_str in bot_strings:
+            if bot_str in agent_str_lower:
+                contains_bot_string = True
+                break
+        if contains_bot_string:
             if agent_str_lower not in known_bots:
                 known_bots.append(agent_str_lower)
                 known_bots.sort()