From 1c25de21c6440b79bb9e90a1b52782eb4fbbb142 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 20 Jun 2021 20:32:06 +0100 Subject: [PATCH 1/6] More reserved names --- utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index ad277004b..fd159e8af 100644 --- a/utils.py +++ b/utils.py @@ -1441,7 +1441,9 @@ def _isReservedName(nickname: str) -> bool: 'shares', 'fonts', 'icons', 'avatars', 'welcome', 'helpimages', 'bookmark', 'bookmarks', 'tlbookmarks', - 'ignores') + 'ignores', 'linksmobile', 'newswiremobile', + 'minimal', 'search', 'eventdelete', + 'searchemoji') if nickname in reservedNames: return True return False From 084da8d2a414432681842a4540f87ff140a9634c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 10:22:24 +0100 Subject: [PATCH 2/6] Cache blocked domains --- blocking.py | 43 +++++++++++++++++++++++++++++++++++++------ daemon.py | 24 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/blocking.py b/blocking.py index 206682efc..40340631c 100644 --- a/blocking.py +++ b/blocking.py @@ -9,6 +9,7 @@ __module_group__ = "ActivityPub" import os import json +import time from datetime import datetime from utils import isAccountDir from utils import getCachedPostFilename @@ -167,7 +168,29 @@ def getDomainBlocklist(baseDir: str) -> str: return blockedStr -def isBlockedDomain(baseDir: str, domain: str) -> bool: +def updateBlockedCache(baseDir: str, + blockedCache: [], + blockedCacheLastUpdated: int, + blockedCacheUpdateSecs: int) -> int: + """Updates the cache of globally blocked domains held in memory + """ + currTime = int(time.time()) + secondsSinceLastUpdate = currTime - blockedCacheLastUpdated + if secondsSinceLastUpdate < blockedCacheUpdateSecs: + return blockedCacheLastUpdated + globalBlockingFilename = baseDir + '/accounts/blocking.txt' + if not os.path.isfile(globalBlockingFilename): + return blockedCacheLastUpdated + with open(globalBlockingFilename, 'r') as fpBlocked: + blockedLines = fpBlocked.readlines() + blockedCache.clear() + for line in blockedLines: + blockedCache.append(line) + return currTime + + +def isBlockedDomain(baseDir: str, domain: str, + blockedCache: [] = None) -> bool: """Is the given domain blocked? """ if '.' not in domain: @@ -186,16 +209,24 @@ def isBlockedDomain(baseDir: str, domain: str) -> bool: allowFilename = baseDir + '/accounts/allowedinstances.txt' if not os.path.isfile(allowFilename): - # instance block list - globalBlockingFilename = baseDir + '/accounts/blocking.txt' - if os.path.isfile(globalBlockingFilename): - with open(globalBlockingFilename, 'r') as fpBlocked: - blockedStr = fpBlocked.read() + if blockedCache: + for blockedStr in blockedCache: if '*@' + domain in blockedStr: return True if shortDomain: if '*@' + shortDomain in blockedStr: return True + else: + # instance block list + globalBlockingFilename = baseDir + '/accounts/blocking.txt' + if os.path.isfile(globalBlockingFilename): + with open(globalBlockingFilename, 'r') as fpBlocked: + blockedStr = fpBlocked.read() + if '*@' + domain in blockedStr: + return True + if shortDomain: + if '*@' + shortDomain in blockedStr: + return True else: # instance allow list if not shortDomain: diff --git a/daemon.py b/daemon.py index 955bebeee..f8b0e91dc 100644 --- a/daemon.py +++ b/daemon.py @@ -114,6 +114,7 @@ from media import attachMedia from media import pathIsImage from media import pathIsVideo from media import pathIsAudio +from blocking import updateBlockedCache from blocking import mutePost from blocking import unmutePost from blocking import setBrochMode @@ -487,6 +488,12 @@ class PubServer(BaseHTTPRequestHandler): # is the User-Agent domain blocked blockedUA = False if not agentDomain.startswith(callingDomain): + self.server.blockedCacheLastUpdated = \ + updateBlockedCache(self.server.baseDir, + self.server.blockedCache, + self.server.blockedCacheLastUpdated, + self.server.blockedCacheUpdateSecs) + blockedUA = isBlockedDomain(self.server.baseDir, agentDomain) # if self.server.debug: if blockedUA: @@ -1212,6 +1219,13 @@ class PubServer(BaseHTTPRequestHandler): messageDomain, messagePort = \ getDomainFromActor(messageJson['actor']) + + self.server.blockedCacheLastUpdated = \ + updateBlockedCache(self.server.baseDir, + self.server.blockedCache, + self.server.blockedCacheLastUpdated, + self.server.blockedCacheUpdateSecs) + if isBlockedDomain(self.server.baseDir, messageDomain): print('POST from blocked domain ' + messageDomain) self._400() @@ -15180,6 +15194,16 @@ def runDaemon(userAgentsBlocked: [], # contains threads used to send posts to followers httpd.followersThreads = [] + # create a cache of blocked domains in memory. + # This limits the amount of slow disk reads which need to be done + httpd.blockedCache = [] + httpd.blockedCacheLastUpdated = 0 + httpd.blockedCacheUpdateSecs = 60 + httpd.blockedCacheLastUpdated = \ + updateBlockedCache(baseDir, httpd.blockedCache, + httpd.blockedCacheLastUpdated, + httpd.blockedCacheUpdateSecs) + # cache to store css files httpd.cssCache = {} From 2e9d1c93d910cd630e980e8b11461fc8e3bd4a1c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 10:26:13 +0100 Subject: [PATCH 3/6] Remove newlines --- blocking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocking.py b/blocking.py index 40340631c..abdb5309a 100644 --- a/blocking.py +++ b/blocking.py @@ -185,7 +185,7 @@ def updateBlockedCache(baseDir: str, blockedLines = fpBlocked.readlines() blockedCache.clear() for line in blockedLines: - blockedCache.append(line) + blockedCache.append(line.replace('\n', '')) return currTime From 67ce8e7ece7d14f00942fa438c21b57c41f8eb41 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 10:27:40 +0100 Subject: [PATCH 4/6] Use blocked cache --- daemon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index f8b0e91dc..d94daba49 100644 --- a/daemon.py +++ b/daemon.py @@ -494,7 +494,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.blockedCacheLastUpdated, self.server.blockedCacheUpdateSecs) - blockedUA = isBlockedDomain(self.server.baseDir, agentDomain) + blockedUA = isBlockedDomain(self.server.baseDir, agentDomain, + self.server.blockedCache) # if self.server.debug: if blockedUA: print('Blocked User agent: ' + agentDomain) @@ -1226,7 +1227,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.blockedCacheLastUpdated, self.server.blockedCacheUpdateSecs) - if isBlockedDomain(self.server.baseDir, messageDomain): + if isBlockedDomain(self.server.baseDir, messageDomain, + self.server.blockedCache): print('POST from blocked domain ' + messageDomain) self._400() self.server.POSTbusy = False From 9cb89aea90adb8779e7845965fb5ccdbea8037c9 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 10:38:06 +0100 Subject: [PATCH 5/6] Slower blocking cache update --- daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index d94daba49..b53021918 100644 --- a/daemon.py +++ b/daemon.py @@ -15200,7 +15200,7 @@ def runDaemon(userAgentsBlocked: [], # This limits the amount of slow disk reads which need to be done httpd.blockedCache = [] httpd.blockedCacheLastUpdated = 0 - httpd.blockedCacheUpdateSecs = 60 + httpd.blockedCacheUpdateSecs = 120 httpd.blockedCacheLastUpdated = \ updateBlockedCache(baseDir, httpd.blockedCache, httpd.blockedCacheLastUpdated, From 727ba3da6ff5875e778426fc7bdbc42480b8a97b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 10:40:43 +0100 Subject: [PATCH 6/6] Handle clock resets --- blocking.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/blocking.py b/blocking.py index abdb5309a..53346c136 100644 --- a/blocking.py +++ b/blocking.py @@ -175,6 +175,9 @@ def updateBlockedCache(baseDir: str, """Updates the cache of globally blocked domains held in memory """ currTime = int(time.time()) + if blockedCacheLastUpdated > currTime: + print('WARN: Cache updated in the future') + blockedCacheLastUpdated = 0 secondsSinceLastUpdate = currTime - blockedCacheLastUpdated if secondsSinceLastUpdate < blockedCacheUpdateSecs: return blockedCacheLastUpdated