From 784aca35f311f39d20857bbe1d4633b5106171aa Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 28 Jun 2021 18:28:14 +0100 Subject: [PATCH 1/2] Don't block crawlers on news instances --- daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon.py b/daemon.py index b3904bf63..6df796b8f 100644 --- a/daemon.py +++ b/daemon.py @@ -466,6 +466,8 @@ class PubServer(BaseHTTPRequestHandler): # is this a web crawler? If so the block it agentStrLower = agentStr.lower() if 'bot/' in agentStrLower or 'bot-' in agentStrLower: + if if self.server.newsInstance: + return False print('Blocked Crawler: ' + agentStr) return True # get domain name from User-Agent From df3de68e52ef2b51d208d98f862ee982f0964d83 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 28 Jun 2021 18:28:57 +0100 Subject: [PATCH 2/2] Don't block crawlers on news instances --- daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 6df796b8f..e02a57147 100644 --- a/daemon.py +++ b/daemon.py @@ -466,7 +466,7 @@ class PubServer(BaseHTTPRequestHandler): # is this a web crawler? If so the block it agentStrLower = agentStr.lower() if 'bot/' in agentStrLower or 'bot-' in agentStrLower: - if if self.server.newsInstance: + if self.server.newsInstance: return False print('Blocked Crawler: ' + agentStr) return True