Reject obvious bots

2021-06-20 17:10:08 +01:00 · 2021-06-20 17:10:08 +01:00 · b45d034332
parent f5930daf41
commit b45d034332
1 changed files with 6 additions and 2 deletions
--- a/daemon.py
+++ b/daemon.py
@ -458,10 +458,14 @@ class PubServer(BaseHTTPRequestHandler):
        """
        agentDomain = None
        if self.headers.get('User-Agent'):
-            agentDomain = userAgentDomain(self.headers['User-Agent'],
-                                          self.server.debug)
+            agentStr = self.headers['User-Agent']
+            if 'bot/' in agentStr or 'bot-' in agentStr.lower():
+                print('Crawler: ' + agentStr)
+                return True
+            agentDomain = userAgentDomain(agentStr, self.server.debug)
        else:
            return True
+
        blockedUA = False
        if not agentDomain:
            if self.server.userAgentDomainRequired: