From 7d6b6d95fb67d742cc21c7ace491589a77ab44db Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 15:32:17 +0100 Subject: [PATCH 01/12] Additional ise of blocked domains cache --- blocking.py | 33 +++++++++++++++++++++++---------- daemon.py | 10 +++++++++- inbox.py | 11 +++++++---- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/blocking.py b/blocking.py index 2902af43f..364fc068c 100644 --- a/blocking.py +++ b/blocking.py @@ -243,19 +243,33 @@ def isBlockedDomain(baseDir: str, domain: str, def isBlocked(baseDir: str, nickname: str, domain: str, - blockNickname: str, blockDomain: str) -> bool: + blockNickname: str, blockDomain: str, + blockedCache: [] = None) -> bool: """Is the given nickname blocked? """ if isEvil(blockDomain): return True - globalBlockingFilename = baseDir + '/accounts/blocking.txt' - if os.path.isfile(globalBlockingFilename): - if '*@' + blockDomain in open(globalBlockingFilename).read(): - return True - if blockNickname: - blockHandle = blockNickname + '@' + blockDomain - if blockHandle in open(globalBlockingFilename).read(): + + blockHandle = None + if blockNickname and blockDomain: + blockHandle = blockNickname + '@' + blockDomain + + if blockedCache: + for blockedStr in blockedCache: + if '*@' + domain in blockedStr: return True + if blockHandle: + if blockHandle in blockedStr: + return True + else: + globalBlockingFilename = baseDir + '/accounts/blocking.txt' + if os.path.isfile(globalBlockingFilename): + if '*@' + blockDomain in open(globalBlockingFilename).read(): + return True + if blockHandle: + if blockHandle in open(globalBlockingFilename).read(): + return True + allowFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/allowedinstances.txt' if os.path.isfile(allowFilename): @@ -266,8 +280,7 @@ def isBlocked(baseDir: str, nickname: str, domain: str, if os.path.isfile(blockingFilename): if '*@' + blockDomain in open(blockingFilename).read(): return True - if blockNickname: - blockHandle = blockNickname + '@' + blockDomain + if blockHandle: if blockHandle in open(blockingFilename).read(): return True return False diff --git a/daemon.py b/daemon.py index b53021918..0cfd93532 100644 --- a/daemon.py +++ b/daemon.py @@ -1277,6 +1277,13 @@ class PubServer(BaseHTTPRequestHandler): beginSaveTime = time.time() # save the json for later queue processing messageBytesDecoded = messageBytes.decode('utf-8') + + self.server.blockedCacheLastUpdated = \ + updateBlockedCache(self.server.baseDir, + self.server.blockedCache, + self.server.blockedCacheLastUpdated, + self.server.blockedCacheUpdateSecs) + queueFilename = \ savePostToInboxQueue(self.server.baseDir, self.server.httpPrefix, @@ -1286,7 +1293,8 @@ class PubServer(BaseHTTPRequestHandler): messageBytesDecoded, headersDict, self.path, - self.server.debug) + self.server.debug, + self.server.blockedCache) if queueFilename: # add json to the queue if queueFilename not in self.server.inboxQueue: diff --git a/inbox.py b/inbox.py index f73a9eae1..9e9751f26 100644 --- a/inbox.py +++ b/inbox.py @@ -353,7 +353,8 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, originalPostJsonObject: {}, messageBytes: str, httpHeaders: {}, - postPath: str, debug: bool) -> str: + postPath: str, debug: bool, + blockedCache: []) -> str: """Saves the give json to the inbox queue for the person keyId specifies the actor sending the post """ @@ -384,7 +385,8 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, pprint(postJsonObject) print('No post Domain in actor') return None - if isBlocked(baseDir, nickname, domain, postNickname, postDomain): + if isBlocked(baseDir, nickname, domain, + postNickname, postDomain, blockedCache): if debug: print('DEBUG: post from ' + postNickname + ' blocked') return None @@ -398,7 +400,7 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, postJsonObject['object']['inReplyTo'] replyDomain, replyPort = \ getDomainFromActor(inReplyTo) - if isBlockedDomain(baseDir, replyDomain): + if isBlockedDomain(baseDir, replyDomain, blockedCache): if debug: print('WARN: post contains reply from ' + str(actor) + @@ -409,7 +411,8 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, getNicknameFromActor(inReplyTo) if replyNickname and replyDomain: if isBlocked(baseDir, nickname, domain, - replyNickname, replyDomain): + replyNickname, replyDomain, + blockedCache): if debug: print('WARN: post contains reply from ' + str(actor) + From 7c9fc8dfbb8a9b9e72f078a5f598afa335ba3bd1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 15:39:05 +0100 Subject: [PATCH 02/12] Simpler update of blocking cache --- blocking.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blocking.py b/blocking.py index 364fc068c..72ce46958 100644 --- a/blocking.py +++ b/blocking.py @@ -186,9 +186,12 @@ def updateBlockedCache(baseDir: str, return blockedCacheLastUpdated with open(globalBlockingFilename, 'r') as fpBlocked: blockedLines = fpBlocked.readlines() + # remove newlines + for index in range(len(blockedLines)): + blockedLines[index] = blockedLines[index].replace('\n', '') + # update the cache blockedCache.clear() - for line in blockedLines: - blockedCache.append(line.replace('\n', '')) + blockedCache += blockedLines return currTime From bbfb92d36be227d50f3002883665a98f735e0b1c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 15:59:35 +0100 Subject: [PATCH 03/12] Remove newline --- city.py | 1 + 1 file changed, 1 insertion(+) diff --git a/city.py b/city.py index 8edf14e1b..1726966d3 100644 --- a/city.py +++ b/city.py @@ -130,6 +130,7 @@ def _getCityPulse(currTimeOfDay, decoySeed: int) -> (float, float): def parseNogoString(nogoLine: str) -> []: """Parses a line from locations_nogo.txt and returns the polygon """ + nogoLine = nogoLine.replace('\n', '').replace('\r', '') polygonStr = nogoLine.split(':', 1)[1] if ';' in polygonStr: pts = polygonStr.split(';') From 971ace14a84c32e36b34cef5f8d72dda0c051fde Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 16:56:55 +0100 Subject: [PATCH 04/12] Support broch mode within general blocking function --- blocking.py | 66 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/blocking.py b/blocking.py index 72ce46958..86a6c052d 100644 --- a/blocking.py +++ b/blocking.py @@ -195,6 +195,18 @@ def updateBlockedCache(baseDir: str, return currTime +def _getShortDomain(domain: str) -> str: + """ by checking a shorter version we can thwart adversaries + who constantly change their subdomain + e.g. subdomain123.mydomain.com becomes mydomain.com + """ + sections = domain.split('.') + noOfSections = len(sections) + if noOfSections > 2: + return sections[noOfSections-2] + '.' + sections[-1] + return None + + def isBlockedDomain(baseDir: str, domain: str, blockedCache: [] = None) -> bool: """Is the given domain blocked? @@ -205,13 +217,7 @@ def isBlockedDomain(baseDir: str, domain: str, if isEvil(domain): return True - # by checking a shorter version we can thwart adversaries - # who constantly change their subdomain - sections = domain.split('.') - noOfSections = len(sections) - shortDomain = None - if noOfSections > 2: - shortDomain = domain[noOfSections-2] + '.' + domain[noOfSections-1] + shortDomain = _getShortDomain(domain) if not brochModeIsActive(baseDir): if blockedCache: @@ -257,29 +263,43 @@ def isBlocked(baseDir: str, nickname: str, domain: str, if blockNickname and blockDomain: blockHandle = blockNickname + '@' + blockDomain - if blockedCache: - for blockedStr in blockedCache: - if '*@' + domain in blockedStr: - return True - if blockHandle: - if blockHandle in blockedStr: + if not brochModeIsActive(baseDir): + # instance level block list + if blockedCache: + for blockedStr in blockedCache: + if '*@' + domain in blockedStr: return True + if blockHandle: + if blockHandle in blockedStr: + return True + else: + globalBlockingFilename = baseDir + '/accounts/blocking.txt' + if os.path.isfile(globalBlockingFilename): + if '*@' + blockDomain in open(globalBlockingFilename).read(): + return True + if blockHandle: + if blockHandle in open(globalBlockingFilename).read(): + return True else: - globalBlockingFilename = baseDir + '/accounts/blocking.txt' - if os.path.isfile(globalBlockingFilename): - if '*@' + blockDomain in open(globalBlockingFilename).read(): + # instance allow list + allowFilename = baseDir + '/accounts/allowedinstances.txt' + shortDomain = _getShortDomain(blockDomain) + if not shortDomain: + if blockDomain not in open(allowFilename).read(): + return True + else: + if shortDomain not in open(allowFilename).read(): return True - if blockHandle: - if blockHandle in open(globalBlockingFilename).read(): - return True - allowFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/allowedinstances.txt' + # account level allow list + accountDir = baseDir + '/accounts/' + nickname + '@' + domain + allowFilename = accountDir + '/allowedinstances.txt' if os.path.isfile(allowFilename): if blockDomain not in open(allowFilename).read(): return True - blockingFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/blocking.txt' + + # account level block list + blockingFilename = accountDir + '/blocking.txt' if os.path.isfile(blockingFilename): if '*@' + blockDomain in open(blockingFilename).read(): return True From 7209d23ae74b3d36b9495a04ce6f0ee461329c83 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 17:32:28 +0100 Subject: [PATCH 05/12] Show more information about errors --- daemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon.py b/daemon.py index 0cfd93532..937241d8e 100644 --- a/daemon.py +++ b/daemon.py @@ -14842,6 +14842,7 @@ class EpicyonServer(ThreadingHTTPServer): print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e)) pass else: + print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e)) return HTTPServer.handle_error(self, request, client_address) From 31a6dbd6574a90209690e96316fc3a7c861367a9 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 17:38:00 +0100 Subject: [PATCH 06/12] Handle broken pipe error --- daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon.py b/daemon.py index 937241d8e..223eace39 100644 --- a/daemon.py +++ b/daemon.py @@ -14841,6 +14841,8 @@ class EpicyonServer(ThreadingHTTPServer): if e.errno != errno.ECONNRESET: print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e)) pass + elif cls is BrokenPipeError: + pass else: print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e)) return HTTPServer.handle_error(self, request, client_address) From 9c18e7042e76c6a1282b4f0ab86c58eeeaf197d4 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 22:02:03 +0100 Subject: [PATCH 07/12] File storage functions --- auth.py | 26 +++---- blocking.py | 6 +- bookmarks.py | 11 +-- categories.py | 7 +- daemon.py | 158 +++++++++++++++---------------------------- desktop_client.py | 6 +- epicyon.py | 9 +-- follow.py | 14 ++-- followingCalendar.py | 13 ++-- git.py | 7 +- happening.py | 31 ++++----- inbox.py | 63 ++++------------- media.py | 14 ++-- migrate.py | 9 +-- newsdaemon.py | 24 ++----- person.py | 30 +++----- petnames.py | 10 ++- posts.py | 50 +++----------- question.py | 21 +++--- shares.py | 11 ++- storage.py | 56 +++++++++++++++ tests.py | 28 ++++---- theme.py | 22 +++--- utils.py | 36 ++++------ webapp_post.py | 15 ++-- webapp_utils.py | 4 +- webapp_welcome.py | 6 +- 27 files changed, 267 insertions(+), 420 deletions(-) create mode 100644 storage.py diff --git a/auth.py b/auth.py index 87b94e5e2..c62dc8236 100644 --- a/auth.py +++ b/auth.py @@ -15,6 +15,7 @@ import secrets import datetime from utils import isSystemAccount from utils import hasUsersPath +from storage import storeValue def _hashPassword(password: str) -> str: @@ -175,8 +176,7 @@ def storeBasicCredentials(baseDir: str, nickname: str, password: str) -> bool: with open(passwordFile, 'a+') as passfile: passfile.write(storeStr + '\n') else: - with open(passwordFile, 'w+') as passfile: - passfile.write(storeStr + '\n') + storeValue(passwordFile, storeStr, 'write') return True @@ -240,18 +240,14 @@ def recordLoginFailure(baseDir: str, ipAddress: str, return failureLog = baseDir + '/accounts/loginfailures.log' - writeType = 'a+' + writeType = 'append' if not os.path.isfile(failureLog): - writeType = 'w+' + writeType = 'writeonly' currTime = datetime.datetime.utcnow() - try: - with open(failureLog, writeType) as fp: - # here we use a similar format to an ssh log, so that - # systems such as fail2ban can parse it - fp.write(currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + - 'ip-127-0-0-1 sshd[20710]: ' + - 'Disconnecting invalid user epicyon ' + - ipAddress + ' port 443: ' + - 'Too many authentication failures [preauth]\n') - except BaseException: - pass + logLineStr = \ + currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + \ + 'ip-127-0-0-1 sshd[20710]: ' + \ + 'Disconnecting invalid user epicyon ' + \ + ipAddress + ' port 443: ' + \ + 'Too many authentication failures [preauth]\n' + storeValue(failureLog, logLineStr, writeType) diff --git a/blocking.py b/blocking.py index 86a6c052d..a90a63953 100644 --- a/blocking.py +++ b/blocking.py @@ -25,6 +25,7 @@ from utils import locatePost from utils import evilIncarnate from utils import getDomainFromActor from utils import getNicknameFromActor +from storage import storeValue def addGlobalBlock(baseDir: str, @@ -493,10 +494,7 @@ def mutePost(baseDir: str, nickname: str, domain: str, port: int, if os.path.isfile(cachedPostFilename): os.remove(cachedPostFilename) - muteFile = open(postFilename + '.muted', 'w+') - if muteFile: - muteFile.write('\n') - muteFile.close() + if storeValue(postFilename + '.muted', '\n', 'writeonly'): print('MUTE: ' + postFilename + '.muted file added') # if the post is in the recent posts cache then mark it as muted diff --git a/bookmarks.py b/bookmarks.py index 58a772b3d..3858d22e7 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -24,6 +24,7 @@ from utils import loadJson from utils import saveJson from posts import getPersonBox from session import postJson +from storage import storeValue def undoBookmarksCollectionEntry(recentPostsCache: {}, @@ -61,10 +62,7 @@ def undoBookmarksCollectionEntry(recentPostsCache: {}, indexStr = '' with open(bookmarksIndexFilename, 'r') as indexFile: indexStr = indexFile.read().replace(bookmarkIndex + '\n', '') - bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') - if bookmarksIndexFile: - bookmarksIndexFile.write(indexStr) - bookmarksIndexFile.close() + storeValue(bookmarksIndexFilename, indexStr, 'writeonly') if not postJsonObject.get('type'): return @@ -219,10 +217,7 @@ def updateBookmarksCollection(recentPostsCache: {}, print('WARN: Failed to write entry to bookmarks index ' + bookmarksIndexFilename + ' ' + str(e)) else: - bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') - if bookmarksIndexFile: - bookmarksIndexFile.write(bookmarkIndex + '\n') - bookmarksIndexFile.close() + storeValue(bookmarksIndexFilename, bookmarkIndex, 'write') def bookmark(recentPostsCache: {}, diff --git a/categories.py b/categories.py index 70d079a78..67c65ba15 100644 --- a/categories.py +++ b/categories.py @@ -9,6 +9,7 @@ __module_group__ = "RSS Feeds" import os import datetime +from storage import storeValue def getHashtagCategory(baseDir: str, hashtag: str) -> str: @@ -106,8 +107,7 @@ def _updateHashtagCategories(baseDir: str) -> None: categoryListStr += categoryStr + '\n' # save a list of available categories for quick lookup - with open(categoryListFilename, 'w+') as fp: - fp.write(categoryListStr) + storeValue(categoryListFilename, categoryListStr, 'writeonly') def _validHashtagCategory(category: str) -> bool: @@ -153,8 +153,7 @@ def setHashtagCategory(baseDir: str, hashtag: str, category: str, # don't overwrite any existing categories if os.path.isfile(categoryFilename): return False - with open(categoryFilename, 'w+') as fp: - fp.write(category) + if storeValue(categoryFilename, category, 'writeonly'): _updateHashtagCategories(baseDir) return True diff --git a/daemon.py b/daemon.py index 223eace39..db2cffa52 100644 --- a/daemon.py +++ b/daemon.py @@ -300,6 +300,7 @@ from context import hasValidContext from speaker import getSSMLbox from city import getSpoofedCity import os +from storage import storeValue # maximum number of posts to list in outbox feed @@ -674,11 +675,7 @@ class PubServer(BaseHTTPRequestHandler): pass if not etag: etag = sha1(data).hexdigest() # nosec - try: - with open(mediaFilename + '.etag', 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaFilename + '.etag', etag, 'writeonly') if etag: self.send_header('ETag', etag) self.end_headers() @@ -1545,12 +1542,7 @@ class PubServer(BaseHTTPRequestHandler): print('WARN: Unable to read salt for ' + loginNickname + ' ' + str(e)) else: - try: - with open(saltFilename, 'w+') as fp: - fp.write(salt) - except Exception as e: - print('WARN: Unable to save salt for ' + - loginNickname + ' ' + str(e)) + storeValue(saltFilename, salt, 'writeonly') tokenText = loginNickname + loginPassword + salt token = sha256(tokenText.encode('utf-8')).hexdigest() @@ -1559,12 +1551,7 @@ class PubServer(BaseHTTPRequestHandler): tokenFilename = \ baseDir+'/accounts/' + \ loginHandle + '/.token' - try: - with open(tokenFilename, 'w+') as fp: - fp.write(token) - except Exception as e: - print('WARN: Unable to save token for ' + - loginNickname + ' ' + str(e)) + storeValue(tokenFilename, token, 'writeonly') personUpgradeActor(baseDir, None, loginHandle, baseDir + '/accounts/' + @@ -2104,10 +2091,8 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - noNewswireFile = open(newswireBlockedFilename, "w+") - if noNewswireFile: - noNewswireFile.write('\n') - noNewswireFile.close() + if storeValue(newswireBlockedFilename, + '\n', 'writeonly'): refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2140,10 +2125,8 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - noFeaturesFile = open(featuresBlockedFilename, "w+") - if noFeaturesFile: - noFeaturesFile.write('\n') - noFeaturesFile.close() + if storeValue(featuresBlockedFilename, + '\n', 'writeonly'): refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2175,10 +2158,7 @@ class PubServer(BaseHTTPRequestHandler): os.remove(newswireModFilename) else: if os.path.isdir(accountDir): - modNewswireFile = open(newswireModFilename, "w+") - if modNewswireFile: - modNewswireFile.write('\n') - modNewswireFile.close() + storeValue(newswireModFilename, '\n', 'writeonly') usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ '?page=' + str(pageNumber) @@ -3459,10 +3439,7 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('editedLinks'): linksStr = fields['editedLinks'] - linksFile = open(linksFilename, "w+") - if linksFile: - linksFile.write(linksStr) - linksFile.close() + storeValue(linksFilename, linksStr, 'writeonly') else: if os.path.isfile(linksFilename): os.remove(linksFilename) @@ -3474,10 +3451,7 @@ class PubServer(BaseHTTPRequestHandler): aboutStr = fields['editedAbout'] if not dangerousMarkup(aboutStr, allowLocalNetworkAccess): - aboutFile = open(aboutFilename, "w+") - if aboutFile: - aboutFile.write(aboutStr) - aboutFile.close() + storeValue(aboutFilename, aboutStr, 'writeonly') else: if os.path.isfile(aboutFilename): os.remove(aboutFilename) @@ -3486,10 +3460,7 @@ class PubServer(BaseHTTPRequestHandler): TOSStr = fields['editedTOS'] if not dangerousMarkup(TOSStr, allowLocalNetworkAccess): - TOSFile = open(TOSFilename, "w+") - if TOSFile: - TOSFile.write(TOSStr) - TOSFile.close() + storeValue(TOSFilename, TOSStr, 'writeonly') else: if os.path.isfile(TOSFilename): os.remove(TOSFilename) @@ -3664,10 +3635,7 @@ class PubServer(BaseHTTPRequestHandler): extractTextFieldsInPOST(postBytes, boundary, debug) if fields.get('editedNewswire'): newswireStr = fields['editedNewswire'] - newswireFile = open(newswireFilename, "w+") - if newswireFile: - newswireFile.write(newswireStr) - newswireFile.close() + storeValue(newswireFilename, newswireStr, 'writeonly') else: if os.path.isfile(newswireFilename): os.remove(newswireFilename) @@ -3677,8 +3645,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ 'news@' + domain + '/filters.txt' if fields.get('filteredWordsNewswire'): - with open(filterNewswireFilename, 'w+') as filterfile: - filterfile.write(fields['filteredWordsNewswire']) + storeValue(filterNewswireFilename, + fields['filteredWordsNewswire'], 'writeonly') else: if os.path.isfile(filterNewswireFilename): os.remove(filterNewswireFilename) @@ -3687,8 +3655,8 @@ class PubServer(BaseHTTPRequestHandler): hashtagRulesFilename = \ baseDir + '/accounts/hashtagrules.txt' if fields.get('hashtagRulesList'): - with open(hashtagRulesFilename, 'w+') as rulesfile: - rulesfile.write(fields['hashtagRulesList']) + storeValue(hashtagRulesFilename, + fields['hashtagRulesList'], 'writeonly') else: if os.path.isfile(hashtagRulesFilename): os.remove(hashtagRulesFilename) @@ -3698,10 +3666,8 @@ class PubServer(BaseHTTPRequestHandler): newswireTrusted = fields['trustedNewswire'] if not newswireTrusted.endswith('\n'): newswireTrusted += '\n' - trustFile = open(newswireTrustedFilename, "w+") - if trustFile: - trustFile.write(newswireTrusted) - trustFile.close() + storeValue(newswireTrustedFilename, + newswireTrusted, 'writeonly') else: if os.path.isfile(newswireTrustedFilename): os.remove(newswireTrustedFilename) @@ -3787,10 +3753,8 @@ class PubServer(BaseHTTPRequestHandler): citationsStr += citationDate + '\n' # save citations dates, so that they can be added when # reloading the newblog screen - citationsFile = open(citationsFilename, "w+") - if citationsFile: - citationsFile.write(citationsStr) - citationsFile.close() + storeValue(citationsFilename, + citationsStr, 'writeonly') # redirect back to the default timeline self._redirect_headers(actorStr + '/newblog', @@ -4226,8 +4190,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('cityDropdown'): cityFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/city.txt' - with open(cityFilename, 'w+') as fp: - fp.write(fields['cityDropdown']) + storeValue(cityFilename, + fields['cityDropdown'], 'writeonly') # change displayed name if fields.get('displayNickname'): @@ -5000,16 +4964,15 @@ class PubServer(BaseHTTPRequestHandler): if onFinalWelcomeScreen: # initial default setting created via # the welcome screen - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, '\n', 'writeonly') actorChanged = True else: followDMsActive = False if fields.get('followDMs'): if fields['followDMs'] == 'on': followDMsActive = True - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, + '\n', 'writeonly') if not followDMsActive: if os.path.isfile(followDMsFilename): os.remove(followDMsFilename) @@ -5023,9 +4986,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('removeTwitter'): if fields['removeTwitter'] == 'on': removeTwitterActive = True - with open(removeTwitterFilename, - 'w+') as rFile: - rFile.write('\n') + storeValue(removeTwitterFilename, + '\n', 'writeonly') if not removeTwitterActive: if os.path.isfile(removeTwitterFilename): os.remove(removeTwitterFilename) @@ -5043,8 +5005,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('hideLikeButton'): if fields['hideLikeButton'] == 'on': hideLikeButtonActive = True - with open(hideLikeButtonFile, 'w+') as rFile: - rFile.write('\n') + storeValue(hideLikeButtonFile, + '\n', 'writeonly') # remove notify likes selection if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5055,8 +5017,8 @@ class PubServer(BaseHTTPRequestHandler): # notify about new Likes if onFinalWelcomeScreen: # default setting from welcome screen - with open(notifyLikesFilename, 'w+') as rFile: - rFile.write('\n') + storeValue(notifyLikesFilename, + '\n', 'writeonly') actorChanged = True else: notifyLikesActive = False @@ -5064,8 +5026,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['notifyLikes'] == 'on' and \ not hideLikeButtonActive: notifyLikesActive = True - with open(notifyLikesFilename, 'w+') as rFile: - rFile.write('\n') + storeValue(notifyLikesFilename, + '\n', 'writeonly') if not notifyLikesActive: if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5108,8 +5070,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/filters.txt' if fields.get('filteredWords'): - with open(filterFilename, 'w+') as filterfile: - filterfile.write(fields['filteredWords']) + storeValue(filterFilename, + fields['filteredWords'], 'writeonly') else: if os.path.isfile(filterFilename): os.remove(filterFilename) @@ -5120,8 +5082,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/replacewords.txt' if fields.get('switchWords'): - with open(switchFilename, 'w+') as switchfile: - switchfile.write(fields['switchWords']) + storeValue(switchFilename, + fields['switchWords'], 'writeonly') else: if os.path.isfile(switchFilename): os.remove(switchFilename) @@ -5132,8 +5094,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autotags.txt' if fields.get('autoTags'): - with open(autoTagsFilename, 'w+') as autoTagsFile: - autoTagsFile.write(fields['autoTags']) + storeValue(autoTagsFilename, + fields['autoTags'], 'writeonly') else: if os.path.isfile(autoTagsFilename): os.remove(autoTagsFilename) @@ -5144,8 +5106,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autocw.txt' if fields.get('autoCW'): - with open(autoCWFilename, 'w+') as autoCWFile: - autoCWFile.write(fields['autoCW']) + storeValue(autoCWFilename, + fields['autoCW'], 'writeonly') else: if os.path.isfile(autoCWFilename): os.remove(autoCWFilename) @@ -5156,8 +5118,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/blocking.txt' if fields.get('blocked'): - with open(blockedFilename, 'w+') as blockedfile: - blockedfile.write(fields['blocked']) + storeValue(blockedFilename, + fields['blocked'], 'writeonly') else: if os.path.isfile(blockedFilename): os.remove(blockedFilename) @@ -5169,8 +5131,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/dmAllowedinstances.txt' if fields.get('dmAllowedInstances'): - with open(dmAllowedInstancesFilename, 'w+') as aFile: - aFile.write(fields['dmAllowedInstances']) + storeValue(dmAllowedInstancesFilename, + fields['dmAllowedInstances'], 'writeonly') else: if os.path.isfile(dmAllowedInstancesFilename): os.remove(dmAllowedInstancesFilename) @@ -5181,8 +5143,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/allowedinstances.txt' if fields.get('allowedInstances'): - with open(allowedInstancesFilename, 'w+') as aFile: - aFile.write(fields['allowedInstances']) + storeValue(allowedInstancesFilename, + fields['allowedInstances'], 'writeonly') else: if os.path.isfile(allowedInstancesFilename): os.remove(allowedInstancesFilename) @@ -5197,8 +5159,8 @@ class PubServer(BaseHTTPRequestHandler): path.startswith('/users/' + adminNickname + '/'): self.server.peertubeInstances.clear() - with open(peertubeInstancesFile, 'w+') as aFile: - aFile.write(fields['ptInstances']) + storeValue(peertubeInstancesFile, + fields['ptInstances'], 'writeonly') ptInstancesList = \ fields['ptInstances'].split('\n') if ptInstancesList: @@ -5220,8 +5182,9 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/gitprojects.txt' if fields.get('gitProjects'): - with open(gitProjectsFilename, 'w+') as aFile: - aFile.write(fields['gitProjects'].lower()) + projectsStr = fields['gitProjects'].lower() + storeValue(gitProjectsFilename, + projectsStr, 'writeonly') else: if os.path.isfile(gitProjectsFilename): os.remove(gitProjectsFilename) @@ -13157,11 +13120,7 @@ class PubServer(BaseHTTPRequestHandler): with open(mediaFilename, 'rb') as avFile: mediaBinary = avFile.read() etag = sha1(mediaBinary).hexdigest() # nosec - try: - with open(mediaTagFilename, 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaTagFilename, etag, 'writeonly') mediaFileType = mediaFileMimeType(checkPath) self._set_headers_head(mediaFileType, fileLength, @@ -13326,13 +13285,8 @@ class PubServer(BaseHTTPRequestHandler): lastUsedFilename = \ self.server.baseDir + '/accounts/' + \ nickname + '@' + self.server.domain + '/.lastUsed' - try: - lastUsedFile = open(lastUsedFilename, 'w+') - if lastUsedFile: - lastUsedFile.write(str(int(time.time()))) - lastUsedFile.close() - except BaseException: - pass + lastUsedStr = str(int(time.time())) + storeValue(lastUsedFilename, lastUsedStr, 'writeonly') mentionsStr = '' if fields.get('mentions'): diff --git a/desktop_client.py b/desktop_client.py index ce47c52aa..ee7fccf83 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -56,6 +56,7 @@ from bookmarks import sendBookmarkViaServer from bookmarks import sendUndoBookmarkViaServer from delete import sendDeleteViaServer from person import getActorJson +from storage import storeValue def _desktopHelp() -> None: @@ -175,10 +176,7 @@ def _markPostAsRead(actor: str, postId: str, postCategory: str) -> None: except Exception as e: print('WARN: Failed to mark post as read' + str(e)) else: - readFile = open(readPostsFilename, 'w+') - if readFile: - readFile.write(postId + '\n') - readFile.close() + storeValue(readPostsFilename, postId, 'write') def _hasReadPost(actor: str, postId: str, postCategory: str) -> bool: diff --git a/epicyon.py b/epicyon.py index a36e4af93..d4f839135 100644 --- a/epicyon.py +++ b/epicyon.py @@ -88,6 +88,7 @@ from announce import sendAnnounceViaServer from socnet import instancesGraph from migrate import migrateAccounts from desktop_client import runDesktopClient +from storage import storeValue def str2bool(v) -> bool: @@ -759,12 +760,8 @@ if args.socnet: proxyType, args.port, httpPrefix, debug, __version__) - try: - with open('socnet.dot', 'w+') as fp: - fp.write(dotGraph) - print('Saved to socnet.dot') - except BaseException: - pass + if storeValue('socnet.dot', dotGraph, 'writeonly'): + print('Saved to socnet.dot') sys.exit() if args.postsraw: diff --git a/follow.py b/follow.py index f7a48ed10..0d447cf8f 100644 --- a/follow.py +++ b/follow.py @@ -30,6 +30,7 @@ from webfinger import webfingerHandle from auth import createBasicAuthHeader from session import getJson from session import postJson +from storage import storeValue def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: @@ -64,8 +65,7 @@ def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: lastSeenDir + '/' + actor.replace('/', '#') + '.txt' print('lastSeenFilename: ' + lastSeenFilename) if not os.path.isfile(lastSeenFilename): - with open(lastSeenFilename, 'w+') as fp: - fp.write(str(100)) + storeValue(lastSeenFilename, '100', 'writeonly') break @@ -279,8 +279,7 @@ def unfollowAccount(baseDir: str, nickname: str, domain: str, with open(unfollowedFilename, "a+") as f: f.write(handleToUnfollow + '\n') else: - with open(unfollowedFilename, "w+") as f: - f.write(handleToUnfollow + '\n') + storeValue(unfollowedFilename, handleToUnfollow, 'write') return True @@ -607,8 +606,7 @@ def _storeFollowRequest(baseDir: str, print('DEBUG: ' + approveHandleStored + ' is already awaiting approval') else: - with open(approveFollowsFilename, "w+") as fp: - fp.write(approveHandleStored + '\n') + storeValue(approveFollowsFilename, approveHandleStored, 'write') # store the follow request in its own directory # We don't rely upon the inbox because items in there could expire @@ -765,9 +763,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, 'Failed to write entry to followers file ' + str(e)) else: - followersFile = open(followersFilename, "w+") - followersFile.write(approveHandle + '\n') - followersFile.close() + storeValue(followersFilename, approveHandle, 'write') print('Beginning follow accept') return followedAccountAccepts(session, baseDir, httpPrefix, diff --git a/followingCalendar.py b/followingCalendar.py index 650687e14..3b4665dbe 100644 --- a/followingCalendar.py +++ b/followingCalendar.py @@ -8,6 +8,7 @@ __status__ = "Production" __module_group__ = "Calendar" import os +from storage import storeValue def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, @@ -30,8 +31,7 @@ def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, # create a new calendar file from the following file with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') return handle + '\n' in open(calendarFilename).read() @@ -75,8 +75,7 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() if add: - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles + handle + '\n') + storeValue(calendarFilename, followingHandles + handle, 'write') # already in the calendar file? if handle + '\n' in followingHandles: @@ -86,16 +85,14 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, return # remove from calendar file followingHandles = followingHandles.replace(handle + '\n', '') - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') else: print(handle + ' not in followingCalendar.txt') # not already in the calendar file if add: # append to the list of handles followingHandles += handle + '\n' - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') def addPersonToCalendar(baseDir: str, nickname: str, domain: str, diff --git a/git.py b/git.py index 71fdc1442..6f66d9c3e 100644 --- a/git.py +++ b/git.py @@ -9,6 +9,7 @@ __module_group__ = "ActivityPub" import os import html +from storage import storeValue def _gitFormatContent(content: str) -> str: @@ -211,12 +212,10 @@ def receiveGitPatch(baseDir: str, nickname: str, domain: str, return False patchStr = \ _gitAddFromHandle(patchStr, '@' + fromNickname + '@' + fromDomain) - with open(patchFilename, 'w+') as patchFile: - patchFile.write(patchStr) + if storeValue(patchFilename, patchStr, 'writeonly'): patchNotifyFilename = \ baseDir + '/accounts/' + \ nickname + '@' + domain + '/.newPatchContent' - with open(patchNotifyFilename, 'w+') as patchFile: - patchFile.write(patchStr) + if storeValue(patchNotifyFilename, patchStr, 'writeonly'): return True return False diff --git a/happening.py b/happening.py index f1443164a..730b45162 100644 --- a/happening.py +++ b/happening.py @@ -16,6 +16,7 @@ from utils import isPublicPost from utils import loadJson from utils import saveJson from utils import locatePost +from storage import storeValue def _validUuid(testUuid: str, version=4): @@ -36,12 +37,7 @@ def _removeEventFromTimeline(eventId: str, tlEventsFilename: str) -> None: return with open(tlEventsFilename, 'r') as fp: eventsTimeline = fp.read().replace(eventId + '\n', '') - try: - with open(tlEventsFilename, 'w+') as fp2: - fp2.write(eventsTimeline) - except BaseException: - print('ERROR: unable to save events timeline') - pass + storeValue(tlEventsFilename, eventsTimeline, 'writeonly') def saveEventPost(baseDir: str, handle: str, postId: str, @@ -105,9 +101,7 @@ def saveEventPost(baseDir: str, handle: str, postId: str, tlEventsFilename + ' ' + str(e)) return False else: - tlEventsFile = open(tlEventsFilename, 'w+') - tlEventsFile.write(eventId + '\n') - tlEventsFile.close() + storeValue(tlEventsFilename, eventId, 'write') # create a directory for the calendar year if not os.path.isdir(calendarPath + '/' + str(eventYear)): @@ -134,17 +128,16 @@ def saveEventPost(baseDir: str, handle: str, postId: str, # a new event has been added calendarNotificationFilename = \ baseDir + '/accounts/' + handle + '/.newCalendar' - calendarNotificationFile = \ - open(calendarNotificationFilename, 'w+') - if not calendarNotificationFile: + calEventStr = \ + '/calendar?year=' + \ + str(eventYear) + \ + '?month=' + \ + str(eventMonthNumber) + \ + '?day=' + \ + str(eventDayOfMonth) + if not storeValue(calendarNotificationFilename, + calEventStr, 'write'): return False - calendarNotificationFile.write('/calendar?year=' + - str(eventYear) + - '?month=' + - str(eventMonthNumber) + - '?day=' + - str(eventDayOfMonth)) - calendarNotificationFile.close() return True diff --git a/inbox.py b/inbox.py index 9e9751f26..0c24e586c 100644 --- a/inbox.py +++ b/inbox.py @@ -83,6 +83,7 @@ from categories import guessHashtagCategory from context import hasValidContext from speaker import updateSpeaker from announce import isSelfAnnounce +from storage import storeValue def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -127,10 +128,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: daysSinceEpoch = daysDiff.days tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n' if not os.path.isfile(tagsFilename): - tagsFile = open(tagsFilename, "w+") - if tagsFile: - tagsFile.write(tagline) - tagsFile.close() + storeValue(tagsFilename, tagline, 'write') else: if postUrl not in open(tagsFilename).read(): try: @@ -1460,10 +1458,7 @@ def _receiveAnnounce(recentPostsCache: {}, postJsonObject, personCache, translate, lookupActor, themeName) - ttsFile = open(postFilename + '.tts', "w+") - if ttsFile: - ttsFile.write('\n') - ttsFile.close() + storeValue(postFilename + '.tts', '\n', 'writeonly') if debug: print('DEBUG: Obtaining actor for announce post ' + @@ -1642,15 +1637,9 @@ def populateReplies(baseDir: str, httpPrefix: str, domain: str, if numLines > maxReplies: return False if messageId not in open(postRepliesFilename).read(): - repliesFile = open(postRepliesFilename, 'a+') - if repliesFile: - repliesFile.write(messageId + '\n') - repliesFile.close() + storeValue(postRepliesFilename, messageId, 'append') else: - repliesFile = open(postRepliesFilename, 'w+') - if repliesFile: - repliesFile.write(messageId + '\n') - repliesFile.close() + storeValue(postRepliesFilename, messageId, 'write') return True @@ -1814,8 +1803,7 @@ def _dmNotify(baseDir: str, handle: str, url: str) -> None: return dmFile = accountDir + '/.newDM' if not os.path.isfile(dmFile): - with open(dmFile, 'w+') as fp: - fp.write(url) + storeValue(dmFile, url, 'writeonly') def _alreadyLiked(baseDir: str, nickname: str, domain: str, @@ -1895,20 +1883,8 @@ def _likeNotify(baseDir: str, domain: str, onionDomain: str, prevLikeStr = fp.read() if prevLikeStr == likeStr: return - try: - with open(prevLikeFile, 'w+') as fp: - fp.write(likeStr) - except BaseException: - print('ERROR: unable to save previous like notification ' + - prevLikeFile) - pass - try: - with open(likeFile, 'w+') as fp: - fp.write(likeStr) - except BaseException: - print('ERROR: unable to write like notification file ' + - likeFile) - pass + storeValue(prevLikeFile, likeStr, 'writeonly') + storeValue(likeFile, likeStr, 'writeonly') def _replyNotify(baseDir: str, handle: str, url: str) -> None: @@ -1919,8 +1895,7 @@ def _replyNotify(baseDir: str, handle: str, url: str) -> None: return replyFile = accountDir + '/.newReply' if not os.path.isfile(replyFile): - with open(replyFile, 'w+') as fp: - fp.write(url) + storeValue(replyFile, url, 'writeonly') def _gitPatchNotify(baseDir: str, handle: str, @@ -1934,8 +1909,7 @@ def _gitPatchNotify(baseDir: str, handle: str, patchFile = accountDir + '/.newPatch' subject = subject.replace('[PATCH]', '').strip() handle = '@' + fromNickname + '@' + fromDomain - with open(patchFile, 'w+') as fp: - fp.write('git ' + handle + ' ' + subject) + storeValue(patchFile, 'git ' + handle + ' ' + subject, 'writeonly') def _groupHandle(baseDir: str, handle: str) -> bool: @@ -2106,13 +2080,7 @@ def inboxUpdateIndex(boxname: str, baseDir: str, handle: str, except Exception as e: print('WARN: Failed to write entry to index ' + str(e)) else: - try: - indexFile = open(indexFilename, 'w+') - if indexFile: - indexFile.write(destinationFilename + '\n') - indexFile.close() - except Exception as e: - print('WARN: Failed to write initial entry to index ' + str(e)) + storeValue(indexFilename, destinationFilename, 'write') return False @@ -2145,8 +2113,8 @@ def _updateLastSeen(baseDir: str, handle: str, actor: str) -> None: if int(daysSinceEpochFile) == daysSinceEpoch: # value hasn't changed, so we can save writing anything to file return - with open(lastSeenFilename, 'w+') as lastSeenFile: - lastSeenFile.write(str(daysSinceEpoch)) + daysSinceEpochStr = str(daysSinceEpoch) + storeValue(lastSeenFilename, daysSinceEpochStr, 'writeonly') def _bounceDM(senderPostId: str, session, httpPrefix: str, @@ -2590,10 +2558,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, # This enables you to ignore a threat that's getting boring if isReplyToMutedPost: print('MUTE REPLY: ' + destinationFilename) - muteFile = open(destinationFilename + '.muted', 'w+') - if muteFile: - muteFile.write('\n') - muteFile.close() + storeValue(destinationFilename + '.muted', '\n', 'writeonly') # update the indexes for different timelines for boxname in updateIndexList: diff --git a/media.py b/media.py index 8b83d4f0d..5530f60f5 100644 --- a/media.py +++ b/media.py @@ -21,6 +21,7 @@ from shutil import copyfile from shutil import rmtree from shutil import move from city import spoofGeolocation +from storage import storeValue def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None: @@ -73,11 +74,8 @@ def _spoofMetaData(baseDir: str, nickname: str, domain: str, decoySeed = int(fp.read()) else: decoySeed = randint(10000, 10000000000000000) - try: - with open(decoySeedFilename, 'w+') as fp: - fp.write(str(decoySeed)) - except BaseException: - pass + decoySeedStr = str(decoySeed) + storeValue(decoySeedFilename, decoySeedStr, 'writeonly') if os.path.isfile('/usr/bin/exiftool'): print('Spoofing metadata in ' + outputFilename + ' using exiftool') @@ -192,11 +190,7 @@ def _updateEtag(mediaFilename: str) -> None: # calculate hash etag = sha1(data).hexdigest() # nosec # save the hash - try: - with open(mediaFilename + '.etag', 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaFilename + '.etag', etag, 'writeonly') def attachMedia(baseDir: str, httpPrefix: str, diff --git a/migrate.py b/migrate.py index 4cca4b0e8..9b03857fa 100644 --- a/migrate.py +++ b/migrate.py @@ -15,6 +15,7 @@ from blocking import isBlocked from session import getJson from posts import getUserUrl from follow import unfollowAccount +from storage import storeValue def _moveFollowingHandlesForAccount(baseDir: str, nickname: str, domain: str, @@ -148,11 +149,11 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, # save the new handles to the refollow list if os.path.isfile(refollowFilename): - with open(refollowFilename, 'a+') as f: - f.write(movedToHandle + '\n') + storeValue(refollowFilename, + movedToHandle, 'append') else: - with open(refollowFilename, 'w+') as f: - f.write(movedToHandle + '\n') + storeValue(refollowFilename, + movedToHandle, 'write') followersFilename = \ baseDir + '/accounts/' + nickname + '@' + domain + '/followers.txt' diff --git a/newsdaemon.py b/newsdaemon.py index bc8e4f5f0..c816417dd 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -34,6 +34,7 @@ from utils import clearFromPostCaches from utils import dangerousMarkup from inbox import storeHashTags from session import createSession +from storage import storeValue def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: @@ -55,19 +56,13 @@ def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: print('WARN: Failed to write entry to feeds posts index ' + indexFilename + ' ' + str(e)) else: - feedsFile = open(indexFilename, 'w+') - if feedsFile: - feedsFile.write(postId + '\n') - feedsFile.close() + storeValue(indexFilename, postId, 'write') def _saveArrivedTime(baseDir: str, postFilename: str, arrived: str) -> None: """Saves the time when an rss post arrived to a file """ - arrivedFile = open(postFilename + '.arrived', 'w+') - if arrivedFile: - arrivedFile.write(arrived) - arrivedFile.close() + storeValue(postFilename + '.arrived', arrived, 'writeonly') def _removeControlCharacters(content: str) -> str: @@ -409,8 +404,7 @@ def _createNewsMirror(baseDir: str, domain: str, for removePostId in removals: indexContent = \ indexContent.replace(removePostId + '\n', '') - with open(mirrorIndexFilename, "w+") as indexFile: - indexFile.write(indexContent) + storeValue(mirrorIndexFilename, indexContent, 'writeonly') mirrorArticleDir = mirrorDir + '/' + postIdNumber if os.path.isdir(mirrorArticleDir): @@ -435,15 +429,9 @@ def _createNewsMirror(baseDir: str, domain: str, # append the post Id number to the index file if os.path.isfile(mirrorIndexFilename): - indexFile = open(mirrorIndexFilename, "a+") - if indexFile: - indexFile.write(postIdNumber + '\n') - indexFile.close() + storeValue(mirrorIndexFilename, postIdNumber, 'append') else: - indexFile = open(mirrorIndexFilename, "w+") - if indexFile: - indexFile.write(postIdNumber + '\n') - indexFile.close() + storeValue(mirrorIndexFilename, postIdNumber, 'write') return True diff --git a/person.py b/person.py index 0771bd115..9cb0af29b 100644 --- a/person.py +++ b/person.py @@ -52,6 +52,7 @@ from session import createSession from session import getJson from webfinger import webfingerHandle from pprint import pprint +from storage import storeValue def generateRSAKey() -> (str, str): @@ -494,15 +495,13 @@ def createPerson(baseDir: str, nickname: str, domain: str, port: int, if manualFollowerApproval: followDMsFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.followDMs' - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, '\n', 'writeonly') # notify when posts are liked if nickname != 'news': notifyLikesFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.notifyLikes' - with open(notifyLikesFilename, 'w+') as nFile: - nFile.write('\n') + storeValue(notifyLikesFilename, '\n', 'writeonly') theme = getConfigParam(baseDir, 'theme') if not theme: @@ -923,15 +922,9 @@ def suspendAccount(baseDir: str, nickname: str, domain: str) -> None: for suspended in lines: if suspended.strip('\n').strip('\r') == nickname: return - suspendedFile = open(suspendedFilename, 'a+') - if suspendedFile: - suspendedFile.write(nickname + '\n') - suspendedFile.close() + storeValue(suspendedFilename, nickname, 'append') else: - suspendedFile = open(suspendedFilename, 'w+') - if suspendedFile: - suspendedFile.write(nickname + '\n') - suspendedFile.close() + storeValue(suspendedFilename, nickname, 'write') def canRemovePost(baseDir: str, nickname: str, @@ -1132,10 +1125,7 @@ def isPersonSnoozed(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - writeSnoozedFile = open(snoozedFilename, 'w+') - if writeSnoozedFile: - writeSnoozedFile.write(content) - writeSnoozedFile.close() + storeValue(snoozedFilename, content, 'writeonly') if snoozeActor + ' ' in open(snoozedFilename).read(): return True @@ -1185,10 +1175,7 @@ def personUnsnooze(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - writeSnoozedFile = open(snoozedFilename, 'w+') - if writeSnoozedFile: - writeSnoozedFile.write(content) - writeSnoozedFile.close() + storeValue(snoozedFilename, content, 'writeonly') def setPersonNotes(baseDir: str, nickname: str, domain: str, @@ -1204,8 +1191,7 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, if not os.path.isdir(notesDir): os.mkdir(notesDir) notesFilename = notesDir + '/' + handle + '.txt' - with open(notesFilename, 'w+') as notesFile: - notesFile.write(notes) + storeValue(notesFilename, notes, 'writeonly') return True diff --git a/petnames.py b/petnames.py index 21c46095a..52ece3c3f 100644 --- a/petnames.py +++ b/petnames.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from storage import storeValue def setPetName(baseDir: str, nickname: str, domain: str, @@ -40,17 +41,14 @@ def setPetName(baseDir: str, nickname: str, domain: str, else: newPetnamesStr += entry # save the updated petnames file - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(newPetnamesStr) + storeValue(petnamesFilename, newPetnamesStr, 'writeonly') return True # entry does not exist in the petnames file - with open(petnamesFilename, 'a+') as petnamesFile: - petnamesFile.write(entry) + storeValue(petnamesFilename, entry, 'append') return True # first entry - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(entry) + storeValue(petnamesFilename, entry, 'writeonly') return True diff --git a/posts.py b/posts.py index 6e275d45f..05291fedf 100644 --- a/posts.py +++ b/posts.py @@ -71,6 +71,7 @@ from filters import isFiltered from git import convertPostToPatch from linked_data_sig import generateJsonSignature from petnames import resolvePetnames +from storage import storeValue def isModerator(baseDir: str, nickname: str) -> bool: @@ -733,17 +734,7 @@ def _updateHashtagsIndex(baseDir: str, tag: {}, newPostId: str) -> None: tagsFile.write(tagline) tagsFile.close() else: - # prepend to tags index file - if tagline not in open(tagsFilename).read(): - try: - with open(tagsFilename, 'r+') as tagsFile: - content = tagsFile.read() - if tagline not in content: - tagsFile.seek(0, 0) - tagsFile.write(tagline + content) - except Exception as e: - print('WARN: Failed to write entry to tags file ' + - tagsFilename + ' ' + str(e)) + storeValue(tagsFilename, tagline, 'prepend') def _addSchedulePost(baseDir: str, nickname: str, domain: str, @@ -767,10 +758,7 @@ def _addSchedulePost(baseDir: str, nickname: str, domain: str, print('WARN: Failed to write entry to scheduled posts index ' + scheduleIndexFilename + ' ' + str(e)) else: - scheduleFile = open(scheduleIndexFilename, 'w+') - if scheduleFile: - scheduleFile.write(indexStr + '\n') - scheduleFile.close() + storeValue(scheduleIndexFilename, indexStr, 'write') def _appendEventFields(newPost: {}, @@ -1194,10 +1182,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, newPost['moderationStatus'] = 'pending' # save to index file moderationIndexFile = baseDir + '/accounts/moderation.txt' - modFile = open(moderationIndexFile, "a+") - if modFile: - modFile.write(newPostId + '\n') - modFile.close() + storeValue(moderationIndexFile, newPostId, 'append') # If a patch has been posted - i.e. the output from # git format-patch - then convert the activitypub type @@ -1305,10 +1290,7 @@ def pinPost(baseDir: str, nickname: str, domain: str, """ accountDir = baseDir + '/accounts/' + nickname + '@' + domain pinnedFilename = accountDir + '/pinToProfile.txt' - pinFile = open(pinnedFilename, "w+") - if pinFile: - pinFile.write(pinnedContent) - pinFile.close() + storeValue(pinnedFilename, pinnedContent, 'writeonly') def undoPinnedPost(baseDir: str, nickname: str, domain: str) -> None: @@ -1850,11 +1832,7 @@ def createReportPost(baseDir: str, newReportFile = baseDir + '/accounts/' + handle + '/.newReport' if os.path.isfile(newReportFile): continue - try: - with open(newReportFile, 'w+') as fp: - fp.write(toUrl + '/moderation') - except BaseException: - pass + storeValue(newReportFile, toUrl + '/moderation', 'writeonly') return postJsonObject @@ -1898,8 +1876,7 @@ def threadSendPost(session, postJsonStr: str, federationList: [], if debug: # save the log file postLogFilename = baseDir + '/post.log' - with open(postLogFilename, "a+") as logFile: - logFile.write(logStr + '\n') + storeValue(postLogFilename, logStr, 'append') if postResult: if debug: @@ -3452,10 +3429,7 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str, break # save the new index file if len(newIndex) > 0: - indexFile = open(indexFilename, 'w+') - if indexFile: - indexFile.write(newIndex) - indexFile.close() + storeValue(indexFilename, newIndex, 'writeonly') postsInBoxDict = {} postsCtr = 0 @@ -3838,8 +3812,7 @@ def checkDomains(session, baseDir: str, updateFollowerWarnings = True if updateFollowerWarnings and followerWarningStr: - with open(followerWarningFilename, 'w+') as fp: - fp.write(followerWarningStr) + storeValue(followerWarningFilename, followerWarningStr, 'writeonly') if not singleCheck: print(followerWarningStr) @@ -3919,10 +3892,7 @@ def _rejectAnnounce(announceFilename: str, # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): - rejectAnnounceFile = open(announceFilename + '.reject', "w+") - if rejectAnnounceFile: - rejectAnnounceFile.write('\n') - rejectAnnounceFile.close() + storeValue(announceFilename + '.reject', '\n', 'writeonly') def downloadAnnounce(session, baseDir: str, httpPrefix: str, diff --git a/question.py b/question.py index 7b7328f44..5d0e08fc5 100644 --- a/question.py +++ b/question.py @@ -11,6 +11,7 @@ import os from utils import locatePost from utils import loadJson from utils import saveJson +from storage import storeValue def questionUpdateVotes(baseDir: str, nickname: str, domain: str, @@ -67,21 +68,17 @@ def questionUpdateVotes(baseDir: str, nickname: str, domain: str, votersFilename = questionPostFilename.replace('.json', '.voters') if not os.path.isfile(votersFilename): # create a new voters file - votersFile = open(votersFilename, 'w+') - if votersFile: - votersFile.write(replyJson['actor'] + - votersFileSeparator + - foundAnswer + '\n') - votersFile.close() + vStr = replyJson['actor'] + \ + votersFileSeparator + \ + foundAnswer + storeValue(votersFilename, vStr, 'write') else: if replyJson['actor'] not in open(votersFilename).read(): # append to the voters file - votersFile = open(votersFilename, "a+") - if votersFile: - votersFile.write(replyJson['actor'] + - votersFileSeparator + - foundAnswer + '\n') - votersFile.close() + vStr = replyJson['actor'] + \ + votersFileSeparator + \ + foundAnswer + storeValue(votersFilename, vStr, 'append') else: # change an entry in the voters file with open(votersFilename, "r") as votersFile: diff --git a/shares.py b/shares.py index 4fb62daad..f3f1d2adf 100644 --- a/shares.py +++ b/shares.py @@ -20,6 +20,7 @@ from utils import loadJson from utils import saveJson from utils import getImageExtensions from media import processMetaData +from storage import storeValue def getValidSharedItemID(displayName: str) -> str: @@ -161,12 +162,10 @@ def addShare(baseDir: str, newShareFile = accountDir + '/.newShare' if not os.path.isfile(newShareFile): nickname = handle.split('@')[0] - try: - with open(newShareFile, 'w+') as fp: - fp.write(httpPrefix + '://' + domainFull + - '/users/' + nickname + '/tlshares') - except BaseException: - pass + storeValue(newShareFile, + httpPrefix + '://' + domainFull + + '/users/' + nickname + '/tlshares', + 'writeonly') break diff --git a/storage.py b/storage.py new file mode 100644 index 000000000..0417796f0 --- /dev/null +++ b/storage.py @@ -0,0 +1,56 @@ +__filename__ = "storage.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.2.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" +__module_group__ = "storage" + +import os + + +def storeValue(filename: str, lineStr: str, storeType: str) -> bool: + """Stores a line to a file + """ + if not lineStr.endswith('\n'): + if storeType != 'writeonly': + lineStr += '\n' + + if storeType[0] == 'a': + if not os.path.isfile(filename): + storeType = 'write' + + if storeType[0] == 'a': + if not os.path.isfile(filename): + return False + # append + try: + with open(filename, "a+") as fp: + fp.write(lineStr) + return True + except Exception as e: + print('ERROR: unable to append to ' + filename + ' ' + str(e)) + pass + elif storeType[0] == 'w': + # new file + try: + with open(filename, "w+") as fp: + fp.write(lineStr) + return True + except Exception as e: + print('ERROR: unable to write to ' + filename + ' ' + str(e)) + pass + elif storeType[0] == 'p': + # prepend + if lineStr not in open(filename).read(): + try: + with open(filename, 'r+') as fp: + content = fp.read() + if lineStr not in content: + fp.seek(0, 0) + fp.write(lineStr + content) + except Exception as e: + print('WARN: Unable to prepend to ' + + filename + ' ' + str(e)) + return False diff --git a/tests.py b/tests.py index f270ae9e1..8b18368f4 100644 --- a/tests.py +++ b/tests.py @@ -117,6 +117,7 @@ from mastoapiv1 import getNicknameFromMastoApiV1Id from webapp_post import prepareHtmlPostNickname from webapp_utils import markdownToHtml from speaker import speakerReplaceLinks +from storage import storeValue testServerAliceRunning = False testServerBobRunning = False @@ -3167,12 +3168,11 @@ def _testFunctions(): callGraphStr += ' }\n' clusterCtr += 1 callGraphStr += '\n}\n' - with open('epicyon_modules.dot', 'w+') as fp: - fp.write(callGraphStr) - print('Modules call graph saved to epicyon_modules.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + - '-Gsep=+100 -Tx11 epicyon_modules.dot') + assert storeValue('epicyon_modules.dot', callGraphStr, 'writeonly') + print('Modules call graph saved to epicyon_modules.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + + '-Gsep=+100 -Tx11 epicyon_modules.dot') callGraphStr = 'digraph Epicyon {\n\n' callGraphStr += ' size="8,6"; ratio=fill;\n' @@ -3223,12 +3223,11 @@ def _testFunctions(): '" [color=' + modColor + '];\n' callGraphStr += '\n}\n' - with open('epicyon.dot', 'w+') as fp: - fp.write(callGraphStr) - print('Call graph saved to epicyon.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + - '-Gsep=+120 -Tx11 epicyon.dot') + assert storeValue('epicyon.dot', callGraphStr, 'writeonly') + print('Call graph saved to epicyon.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + + '-Gsep=+120 -Tx11 epicyon.dot') def _testLinksWithinPost() -> None: @@ -3883,10 +3882,7 @@ def _testSpoofGeolocation() -> None: kmlStr += '\n' kmlStr += '' - kmlFile = open('unittest_decoy.kml', 'w+') - if kmlFile: - kmlFile.write(kmlStr) - kmlFile.close() + assert storeValue('unittest_decoy.kml', kmlStr, 'writeonly') def _testSkills() -> None: diff --git a/theme.py b/theme.py index 7fdb0d720..5fa2bac01 100644 --- a/theme.py +++ b/theme.py @@ -16,6 +16,7 @@ from shutil import make_archive from shutil import unpack_archive from shutil import rmtree from content import dangerousCSS +from storage import storeValue def importTheme(baseDir: str, filename: str) -> bool: @@ -361,8 +362,7 @@ def _setThemeFromDict(baseDir: str, name: str, continue css = setCSSparam(css, paramName, paramValue) filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') if bgParams.get('login'): _setBackgroundFormat(baseDir, name, 'login', bgParams['login']) @@ -388,8 +388,7 @@ def _setBackgroundFormat(baseDir: str, name: str, with open(cssFilename, 'r') as cssfile: css = cssfile.read() css = css.replace('background.jpg', 'background.' + extension) - with open(cssFilename, 'w+') as cssfile2: - cssfile2.write(css) + storeValue(cssFilename, css, 'writeonly') def enableGrayscale(baseDir: str) -> None: @@ -407,12 +406,10 @@ def enableGrayscale(baseDir: str) -> None: css.replace('body, html {', 'body, html {\n filter: grayscale(100%);') filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') grayscaleFilename = baseDir + '/accounts/.grayscale' if not os.path.isfile(grayscaleFilename): - with open(grayscaleFilename, 'w+') as grayfile: - grayfile.write(' ') + storeValue(grayscaleFilename, ' ', 'writeonly') def disableGrayscale(baseDir: str) -> None: @@ -429,8 +426,7 @@ def disableGrayscale(baseDir: str) -> None: css = \ css.replace('\n filter: grayscale(100%);', '') filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') grayscaleFilename = baseDir + '/accounts/.grayscale' if os.path.isfile(grayscaleFilename): os.remove(grayscaleFilename) @@ -470,8 +466,7 @@ def _setCustomFont(baseDir: str): customFontType + "')") css = setCSSparam(css, "*font-family", "'CustomFont'") filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') def _readVariablesFile(baseDir: str, themeName: str, @@ -739,8 +734,7 @@ def _setClearCacheFlag(baseDir: str) -> None: if not os.path.isdir(baseDir + '/accounts'): return flagFilename = baseDir + '/accounts/.clear_cache' - with open(flagFilename, 'w+') as flagFile: - flagFile.write('\n') + storeValue(flagFilename, '\n', 'writeonly') def setTheme(baseDir: str, name: str, domain: str, diff --git a/utils.py b/utils.py index fd159e8af..e22a2f68d 100644 --- a/utils.py +++ b/utils.py @@ -18,6 +18,7 @@ from pprint import pprint from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes +from storage import storeValue # posts containing these strings will always get screened out, # both incoming and outgoing. @@ -43,9 +44,7 @@ def refreshNewswire(baseDir: str): refreshNewswireFilename = baseDir + '/accounts/.refresh_newswire' if os.path.isfile(refreshNewswireFilename): return - refreshFile = open(refreshNewswireFilename, 'w+') - refreshFile.write('\n') - refreshFile.close() + storeValue(refreshNewswireFilename, '\n', 'writeonly') def getSHA256(msg: str): @@ -490,15 +489,13 @@ def saveJson(jsonObject: {}, filename: str) -> bool: """Saves json to a file """ tries = 0 + storeStr = json.dumps(jsonObject) while tries < 5: - try: - with open(filename, 'w+') as fp: - fp.write(json.dumps(jsonObject)) - return True - except BaseException: - print('WARN: saveJson ' + str(tries)) - time.sleep(1) - tries += 1 + if storeValue(filename, storeStr, 'writeonly'): + return True + print('WARN: saveJson ' + str(tries)) + time.sleep(1) + tries += 1 return False @@ -942,8 +939,7 @@ def _setDefaultPetName(baseDir: str, nickname: str, domain: str, followNickname + '@' + followDomain + '\n' if not os.path.isfile(petnamesFilename): # if there is no existing petnames lookup file - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(petnameLookupEntry) + storeValue(petnamesFilename, petnameLookupEntry, 'writeonly') return with open(petnamesFilename, 'r') as petnamesFile: @@ -1000,8 +996,7 @@ def followPerson(baseDir: str, nickname: str, domain: str, for line in lines: if handleToFollow not in line: newLines += line - with open(unfollowedFilename, 'w+') as f: - f.write(newLines) + storeValue(unfollowedFilename, newLines, 'writeonly') if not os.path.isdir(baseDir + '/accounts'): os.mkdir(baseDir + '/accounts') @@ -1029,8 +1024,7 @@ def followPerson(baseDir: str, nickname: str, domain: str, print('DEBUG: ' + handle + ' creating new following file to follow ' + handleToFollow + ', filename is ' + filename) - with open(filename, 'w+') as f: - f.write(handleToFollow + '\n') + storeValue(filename, handleToFollow, 'write') if followFile.endswith('following.txt'): # Default to adding new follows to the calendar. @@ -1352,8 +1346,7 @@ def deletePost(baseDir: str, httpPrefix: str, # hashtag file os.remove(tagIndexFilename) else: - with open(tagIndexFilename, "w+") as f: - f.write(newlines) + storeValue(tagIndexFilename, newlines, 'writeonly') # remove any replies repliesFilename = postFilename.replace('.json', '.replies') @@ -2198,10 +2191,7 @@ def rejectPostId(baseDir: str, nickname: str, domain: str, if recentPostsCache['html'].get(postUrl): del recentPostsCache['html'][postUrl] - rejectFile = open(postFilename + '.reject', "w+") - if rejectFile: - rejectFile.write('\n') - rejectFile.close() + storeValue(postFilename + '.reject', '\n', 'writeonly') def isDM(postJsonObject: {}) -> bool: diff --git a/webapp_post.py b/webapp_post.py index f1f845b73..e96689be2 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -69,6 +69,7 @@ from webapp_question import insertQuestion from devices import E2EEdecryptMessageFromDevice from webfinger import webfingerHandle from speaker import updateSpeaker +from storage import storeValue def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None: @@ -156,13 +157,7 @@ def _saveIndividualPostAsHtmlToCache(baseDir: str, if not os.path.isdir(htmlPostCacheDir): os.mkdir(htmlPostCacheDir) - try: - with open(cachedPostFilename, 'w+') as fp: - fp.write(postHtml) - return True - except Exception as e: - print('ERROR: saving post to cache ' + str(e)) - return False + return storeValue(cachedPostFilename, postHtml, 'writeonly') def _getPostFromRecentCache(session, @@ -1332,10 +1327,8 @@ def individualPostAsHtml(allowDownloads: bool, postJsonObject, personCache, translate, postJsonObject['actor'], themeName) - ttsFile = open(announceFilename + '.tts', "w+") - if ttsFile: - ttsFile.write('\n') - ttsFile.close() + storeValue(announceFilename + '.tts', + '\n', 'writeonly') isAnnounced = True diff --git a/webapp_utils.py b/webapp_utils.py index 7797d040c..1c611473d 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -20,6 +20,7 @@ from cache import getPersonFromCache from cache import storePersonInCache from content import addHtmlTags from content import replaceEmojiFromTags +from storage import storeValue def _markdownEmphasisHtml(markdown: str) -> str: @@ -1387,5 +1388,4 @@ def setMinimal(baseDir: str, domain: str, nickname: str, if minimal and minimalFileExists: os.remove(minimalFilename) elif not minimal and not minimalFileExists: - with open(minimalFilename, 'w+') as fp: - fp.write('\n') + storeValue(minimalFilename, '\n', 'writeonly') diff --git a/webapp_welcome.py b/webapp_welcome.py index 78e1cf441..9bd0f5111 100644 --- a/webapp_welcome.py +++ b/webapp_welcome.py @@ -14,6 +14,7 @@ from utils import removeHtml from webapp_utils import htmlHeaderWithExternalStyle from webapp_utils import htmlFooter from webapp_utils import markdownToHtml +from storage import storeValue def isWelcomeScreenComplete(baseDir: str, nickname: str, domain: str) -> bool: @@ -34,10 +35,7 @@ def welcomeScreenIsComplete(baseDir: str, if not os.path.isdir(accountPath): return completeFilename = accountPath + '/.welcome_complete' - completeFile = open(completeFilename, 'w+') - if completeFile: - completeFile.write('\n') - completeFile.close() + storeValue(completeFilename, '\n', 'writeonly') def htmlWelcomeScreen(baseDir: str, nickname: str, From feb4286031fcae4a8149547ef5dd810bc0c0bb22 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 22:21:43 +0100 Subject: [PATCH 08/12] Storage functions --- auth.py | 3 +-- blocking.py | 14 +++----------- daemon.py | 5 +---- filters.py | 9 +++------ follow.py | 6 ++---- happening.py | 26 +++++++++++++------------- inbox.py | 10 ++-------- manualapprove.py | 17 +++++------------ person.py | 7 ++----- threads.py | 13 ++++++------- utils.py | 3 +-- 11 files changed, 39 insertions(+), 74 deletions(-) diff --git a/auth.py b/auth.py index c62dc8236..2deb93bf0 100644 --- a/auth.py +++ b/auth.py @@ -173,8 +173,7 @@ def storeBasicCredentials(baseDir: str, nickname: str, password: str) -> bool: os.rename(passwordFile + '.new', passwordFile) else: # append to password file - with open(passwordFile, 'a+') as passfile: - passfile.write(storeStr + '\n') + storeValue(passwordFile, storeStr, 'append') else: storeValue(passwordFile, storeStr, 'write') return True diff --git a/blocking.py b/blocking.py index a90a63953..e0c8ab767 100644 --- a/blocking.py +++ b/blocking.py @@ -40,10 +40,7 @@ def addGlobalBlock(baseDir: str, if blockHandle in open(blockingFilename).read(): return False # block an account handle or domain - blockFile = open(blockingFilename, "a+") - if blockFile: - blockFile.write(blockHandle + '\n') - blockFile.close() + storeValue(blockingFilename, blockHandle, 'append') else: blockHashtag = blockNickname # is the hashtag already blocked? @@ -51,10 +48,7 @@ def addGlobalBlock(baseDir: str, if blockHashtag + '\n' in open(blockingFilename).read(): return False # block a hashtag - blockFile = open(blockingFilename, "a+") - if blockFile: - blockFile.write(blockHashtag + '\n') - blockFile.close() + storeValue(blockingFilename, blockHashtag, 'append') return True @@ -70,9 +64,7 @@ def addBlock(baseDir: str, nickname: str, domain: str, if os.path.isfile(blockingFilename): if blockHandle in open(blockingFilename).read(): return False - blockFile = open(blockingFilename, "a+") - blockFile.write(blockHandle + '\n') - blockFile.close() + storeValue(blockingFilename, blockHandle, 'append') return True diff --git a/daemon.py b/daemon.py index db2cffa52..32ee60d6e 100644 --- a/daemon.py +++ b/daemon.py @@ -432,10 +432,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxReplies, self.server.debug) # record the vote - votesFile = open(votesFilename, 'a+') - if votesFile: - votesFile.write(messageId + '\n') - votesFile.close() + storeValue(votesFilename, messageId, 'append') # ensure that the cached post is removed if it exists, # so that it then will be recreated diff --git a/filters.py b/filters.py index f7dc2193b..53c837bc9 100644 --- a/filters.py +++ b/filters.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from storage import storeValue def addFilter(baseDir: str, nickname: str, domain: str, words: str) -> bool: @@ -17,9 +18,7 @@ def addFilter(baseDir: str, nickname: str, domain: str, words: str) -> bool: if os.path.isfile(filtersFilename): if words in open(filtersFilename).read(): return False - filtersFile = open(filtersFilename, "a+") - filtersFile.write(words + '\n') - filtersFile.close() + storeValue(filtersFilename, words, 'append') return True @@ -35,9 +34,7 @@ def addGlobalFilter(baseDir: str, words: str) -> bool: if os.path.isfile(filtersFilename): if words in open(filtersFilename).read(): return False - filtersFile = open(filtersFilename, "a+") - filtersFile.write(words + '\n') - filtersFile.close() + storeValue(filtersFilename, words, 'append') return True diff --git a/follow.py b/follow.py index 0d447cf8f..9eb158491 100644 --- a/follow.py +++ b/follow.py @@ -276,8 +276,7 @@ def unfollowAccount(baseDir: str, nickname: str, domain: str, if os.path.isfile(unfollowedFilename): if handleToUnfollowLower not in \ open(unfollowedFilename).read().lower(): - with open(unfollowedFilename, "a+") as f: - f.write(handleToUnfollow + '\n') + storeValue(unfollowedFilename, handleToUnfollow, 'append') else: storeValue(unfollowedFilename, handleToUnfollow, 'write') @@ -599,8 +598,7 @@ def _storeFollowRequest(baseDir: str, if os.path.isfile(approveFollowsFilename): if approveHandle not in open(approveFollowsFilename).read(): - with open(approveFollowsFilename, 'a+') as fp: - fp.write(approveHandleStored + '\n') + storeValue(approveFollowsFilename, approveHandleStored, 'append') else: if debug: print('DEBUG: ' + approveHandleStored + diff --git a/happening.py b/happening.py index 730b45162..1bc8685a5 100644 --- a/happening.py +++ b/happening.py @@ -118,11 +118,8 @@ def saveEventPost(baseDir: str, handle: str, postId: str, return False # append the post Id to the file for the calendar month - calendarFile = open(calendarFilename, 'a+') - if not calendarFile: + if not storeValue(calendarFilename, postId, 'append'): return False - calendarFile.write(postId + '\n') - calendarFile.close() # create a file which will trigger a notification that # a new event has been added @@ -245,9 +242,10 @@ def getTodaysEvents(baseDir: str, nickname: str, domain: str, # if some posts have been deleted then regenerate the calendar file if recreateEventsFile: calendarFile = open(calendarFilename, 'w+') - for postId in calendarPostIds: - calendarFile.write(postId + '\n') - calendarFile.close() + if calendarFile: + for postId in calendarPostIds: + calendarFile.write(postId + '\n') + calendarFile.close() return events @@ -362,9 +360,10 @@ def getThisWeeksEvents(baseDir: str, nickname: str, domain: str) -> {}: # if some posts have been deleted then regenerate the calendar file if recreateEventsFile: calendarFile = open(calendarFilename, 'w+') - for postId in calendarPostIds: - calendarFile.write(postId + '\n') - calendarFile.close() + if calendarFile: + for postId in calendarPostIds: + calendarFile.write(postId + '\n') + calendarFile.close() return events @@ -427,9 +426,10 @@ def getCalendarEvents(baseDir: str, nickname: str, domain: str, # if some posts have been deleted then regenerate the calendar file if recreateEventsFile: calendarFile = open(calendarFilename, 'w+') - for postId in calendarPostIds: - calendarFile.write(postId + '\n') - calendarFile.close() + if calendarFile: + for postId in calendarPostIds: + calendarFile.write(postId + '\n') + calendarFile.close() return events diff --git a/inbox.py b/inbox.py index 0c24e586c..e80fc9071 100644 --- a/inbox.py +++ b/inbox.py @@ -2816,10 +2816,7 @@ def _checkJsonSignature(baseDir: str, queueJson: {}) -> (bool, bool): alreadyUnknown = True if not alreadyUnknown: - unknownFile = open(unknownContextsFile, "a+") - if unknownFile: - unknownFile.write(unknownContext + '\n') - unknownFile.close() + storeValue(unknownContextsFile, unknownContext, 'append') else: print('Unrecognized jsonld signature type: ' + jwebsigType) @@ -2834,10 +2831,7 @@ def _checkJsonSignature(baseDir: str, queueJson: {}) -> (bool, bool): alreadyUnknown = True if not alreadyUnknown: - unknownFile = open(unknownSignaturesFile, "a+") - if unknownFile: - unknownFile.write(jwebsigType + '\n') - unknownFile.close() + storeValue(unknownSignaturesFile, jwebsigType, 'append') return hasJsonSignature, jwebsigType diff --git a/manualapprove.py b/manualapprove.py index d89f23a79..7e97543b4 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -12,6 +12,7 @@ from follow import followedAccountAccepts from follow import followedAccountRejects from follow import removeFromFollowRequests from utils import loadJson +from storage import storeValue def manualDenyFollowRequest(session, baseDir: str, @@ -41,9 +42,7 @@ def manualDenyFollowRequest(session, baseDir: str, removeFromFollowRequests(baseDir, nickname, domain, denyHandle, debug) # Store rejected follows - rejectsFile = open(rejectedFollowsFilename, "a+") - rejectsFile.write(denyHandle + '\n') - rejectsFile.close() + storeValue(rejectedFollowsFilename, denyHandle, 'append') denyNickname = denyHandle.split('@')[0] denyDomain = \ @@ -70,13 +69,9 @@ def _approveFollowerHandle(accountDir: str, approveHandle: str) -> None: approvedFilename = accountDir + '/approved.txt' if os.path.isfile(approvedFilename): if approveHandle not in open(approvedFilename).read(): - approvedFile = open(approvedFilename, "a+") - approvedFile.write(approveHandle + '\n') - approvedFile.close() + storeValue(approvedFilename, approveHandle, 'append') else: - approvedFile = open(approvedFilename, "w+") - approvedFile.write(approveHandle + '\n') - approvedFile.close() + storeValue(approvedFilename, approveHandle, 'write') def manualApproveFollowRequest(session, baseDir: str, @@ -201,9 +196,7 @@ def manualApproveFollowRequest(session, baseDir: str, else: print('Manual follow accept: first follower accepted for ' + handle + ' is ' + approveHandleFull) - followersFile = open(followersFilename, "w+") - followersFile.write(approveHandleFull + '\n') - followersFile.close() + storeValue(followersFilename, approveHandleFull, 'write') # only update the follow requests file if the follow is confirmed to be # in followers.txt diff --git a/person.py b/person.py index 9cb0af29b..f9052ff86 100644 --- a/person.py +++ b/person.py @@ -1144,11 +1144,8 @@ def personSnooze(baseDir: str, nickname: str, domain: str, if os.path.isfile(snoozedFilename): if snoozeActor + ' ' in open(snoozedFilename).read(): return - snoozedFile = open(snoozedFilename, "a+") - if snoozedFile: - snoozedFile.write(snoozeActor + ' ' + - str(int(time.time())) + '\n') - snoozedFile.close() + storeStr = snoozeActor + ' ' + str(int(time.time())) + storeValue(snoozedFilename, storeStr, 'append') def personUnsnooze(baseDir: str, nickname: str, domain: str, diff --git a/threads.py b/threads.py index c07de0821..189b41a2f 100644 --- a/threads.py +++ b/threads.py @@ -10,6 +10,7 @@ import threading import sys import time import datetime +from storage import storeValue class threadWithTrace(threading.Thread): @@ -139,10 +140,8 @@ def removeDormantThreads(baseDir: str, threadsList: [], debug: bool, if debug: sendLogFilename = baseDir + '/send.csv' - try: - with open(sendLogFilename, "a+") as logFile: - logFile.write(currTime.strftime("%Y-%m-%dT%H:%M:%SZ") + - ',' + str(noOfActiveThreads) + - ',' + str(len(threadsList)) + '\n') - except BaseException: - pass + storeStr = \ + currTime.strftime("%Y-%m-%dT%H:%M:%SZ") + \ + ',' + str(noOfActiveThreads) + \ + ',' + str(len(threadsList)) + storeValue(sendLogFilename, storeStr, 'append') diff --git a/utils.py b/utils.py index e22a2f68d..85d24364e 100644 --- a/utils.py +++ b/utils.py @@ -951,8 +951,7 @@ def _setDefaultPetName(baseDir: str, nickname: str, domain: str, # petname already exists return # petname doesn't already exist - with open(petnamesFilename, 'a+') as petnamesFile: - petnamesFile.write(petnameLookupEntry) + storeValue(petnamesFilename, petnameLookupEntry, 'append') def followPerson(baseDir: str, nickname: str, domain: str, From ee0ffade9d74f8465338aab59546ef296c8f2a74 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 23:34:46 +0100 Subject: [PATCH 09/12] Reading functions --- blog.py | 6 ++-- bookmarks.py | 8 ++--- categories.py | 18 +++++------ city.py | 4 +-- content.py | 6 ++-- daemon.py | 35 ++++----------------- desktop_client.py | 8 ++--- follow.py | 14 ++++----- followingCalendar.py | 13 +++----- happening.py | 7 +++-- inbox.py | 12 ++++---- manualapprove.py | 5 ++- media.py | 7 +++-- newsdaemon.py | 6 ++-- newswire.py | 10 +++--- person.py | 9 ++---- petnames.py | 13 ++++---- posts.py | 13 +++----- speaker.py | 13 ++++---- storage.py | 16 ++++++++++ theme.py | 41 ++++++++++++------------- utils.py | 70 ++++++++++++++++++++---------------------- webapp_about.py | 6 ++-- webapp_column_left.py | 10 +++--- webapp_column_right.py | 10 +++--- webapp_create_post.py | 56 ++++++++++++++++----------------- 26 files changed, 199 insertions(+), 217 deletions(-) diff --git a/blog.py b/blog.py index 0822aa64a..5acefa981 100644 --- a/blog.py +++ b/blog.py @@ -28,6 +28,7 @@ from utils import firstParagraphFromString from posts import createBlogsTimeline from newswire import rss2Header from newswire import rss2Footer +from storage import readWholeFile def _noOfBlogReplies(baseDir: str, httpPrefix: str, translate: {}, @@ -744,8 +745,9 @@ def htmlEditBlog(mediaInstance: bool, translate: {}, editBlogText = '' + translate['Write your post text below.'] + '' if os.path.isfile(baseDir + '/accounts/newpost.txt'): - with open(baseDir + '/accounts/newpost.txt', 'r') as file: - editBlogText = '

' + file.read() + '

' + newPostStr = readWholeFile(baseDir + '/accounts/newpost.txt') + if newPostStr: + editBlogText = '

' + newPostStr + '

' cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): diff --git a/bookmarks.py b/bookmarks.py index 3858d22e7..f5f55fe96 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -25,6 +25,7 @@ from utils import saveJson from posts import getPersonBox from session import postJson from storage import storeValue +from storage import readWholeFile def undoBookmarksCollectionEntry(recentPostsCache: {}, @@ -59,10 +60,9 @@ def undoBookmarksCollectionEntry(recentPostsCache: {}, bookmarkIndex = bookmarkIndex.replace('\n', '').replace('\r', '') if bookmarkIndex not in open(bookmarksIndexFilename).read(): return - indexStr = '' - with open(bookmarksIndexFilename, 'r') as indexFile: - indexStr = indexFile.read().replace(bookmarkIndex + '\n', '') - storeValue(bookmarksIndexFilename, indexStr, 'writeonly') + indexStr = readWholeFile(bookmarksIndexFilename) + indexStr = indexStr.replace(bookmarkIndex + '\n', '') + storeValue(bookmarksIndexFilename, indexStr, 'writeonly') if not postJsonObject.get('type'): return diff --git a/categories.py b/categories.py index 67c65ba15..32247ccf2 100644 --- a/categories.py +++ b/categories.py @@ -10,6 +10,7 @@ __module_group__ = "RSS Feeds" import os import datetime from storage import storeValue +from storage import readWholeFile def getHashtagCategory(baseDir: str, hashtag: str) -> str: @@ -24,10 +25,9 @@ def getHashtagCategory(baseDir: str, hashtag: str) -> str: if not os.path.isfile(categoryFilename): return '' - with open(categoryFilename, 'r') as fp: - categoryStr = fp.read() - if categoryStr: - return categoryStr + categoryStr = readWholeFile(categoryFilename) + if categoryStr: + return categoryStr return '' @@ -53,12 +53,10 @@ def getHashtagCategories(baseDir: str, hashtag = f.split('.')[0] if len(hashtag) > maxTagLength: continue - with open(categoryFilename, 'r') as fp: - categoryStr = fp.read() - - if not categoryStr: - continue - + categoryStr = readWholeFile(categoryFilename) + if not categoryStr: + continue + if categoryStr: if category: # only return a dictionary for a specific category if categoryStr != category: diff --git a/city.py b/city.py index 1726966d3..12d5ba570 100644 --- a/city.py +++ b/city.py @@ -12,6 +12,7 @@ import datetime import random import math from random import randint +from storage import readWholeFile # states which the simulated city dweller can be in PERSON_SLEEP = 0 @@ -294,8 +295,7 @@ def getSpoofedCity(city: str, baseDir: str, nickname: str, domain: str) -> str: cityFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/city.txt' if os.path.isfile(cityFilename): - with open(cityFilename, 'r') as fp: - city = fp.read().replace('\n', '') + city = readWholeFile(cityFilename).replace('\n', '') return city diff --git a/content.py b/content.py index 22ba06f0b..89821b983 100644 --- a/content.py +++ b/content.py @@ -19,6 +19,7 @@ from utils import dangerousMarkup from utils import isPGPEncrypted from utils import containsPGPPublicKey from petnames import getPetName +from storage import readWholeFile def removeHtmlTag(htmlStr: str, tag: str) -> str: @@ -169,8 +170,9 @@ def dangerousCSS(filename: str, allowLocalNetworkAccess: bool) -> bool: if not os.path.isfile(filename): return False - with open(filename, 'r') as fp: - content = fp.read().lower() + content = readWholeFile(filename) + if content: + content = content.lower() cssMatches = ('behavior:', ':expression', '?php', '.php', 'google', 'regexp', 'localhost', diff --git a/daemon.py b/daemon.py index 32ee60d6e..6ca766959 100644 --- a/daemon.py +++ b/daemon.py @@ -301,6 +301,7 @@ from speaker import getSSMLbox from city import getSpoofedCity import os from storage import storeValue +from storage import readWholeFile # maximum number of posts to list in outbox feed @@ -665,11 +666,7 @@ class PubServer(BaseHTTPRequestHandler): # self.send_header('Cache-Control', 'public, max-age=86400') etag = None if os.path.isfile(mediaFilename + '.etag'): - try: - with open(mediaFilename + '.etag', 'r') as etagFile: - etag = etagFile.read() - except BaseException: - pass + etag = readWholeFile(mediaFilename + '.etag') if not etag: etag = sha1(data).hexdigest() # nosec storeValue(mediaFilename + '.etag', etag, 'writeonly') @@ -690,12 +687,7 @@ class PubServer(BaseHTTPRequestHandler): oldEtag = self.headers['If-None-Match'] if os.path.isfile(mediaFilename + '.etag'): # load the etag from file - currEtag = '' - try: - with open(mediaFilename, 'r') as etagFile: - currEtag = etagFile.read() - except BaseException: - pass + currEtag = readWholeFile(mediaFilename) if oldEtag == currEtag: # The file has not changed return True @@ -1532,12 +1524,7 @@ class PubServer(BaseHTTPRequestHandler): loginNickname + '@' + domain + '/.salt' salt = createPassword(32) if os.path.isfile(saltFilename): - try: - with open(saltFilename, 'r') as fp: - salt = fp.read() - except Exception as e: - print('WARN: Unable to read salt for ' + - loginNickname + ' ' + str(e)) + salt = readWholeFile(saltFilename) else: storeValue(saltFilename, salt, 'writeonly') @@ -13108,11 +13095,7 @@ class PubServer(BaseHTTPRequestHandler): fileLength = os.path.getsize(mediaFilename) mediaTagFilename = mediaFilename + '.etag' if os.path.isfile(mediaTagFilename): - try: - with open(mediaTagFilename, 'r') as etagFile: - etag = etagFile.read() - except BaseException: - pass + etag = readWholeFile(mediaTagFilename) else: with open(mediaFilename, 'rb') as avFile: mediaBinary = avFile.read() @@ -14856,13 +14839,7 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: if not os.path.isfile(tokenFilename): continue nickname = handle.split('@')[0] - token = None - try: - with open(tokenFilename, 'r') as fp: - token = fp.read() - except Exception as e: - print('WARN: Unable to read token for ' + - nickname + ' ' + str(e)) + token = readWholeFile(tokenFilename) if not token: continue tokensDict[nickname] = token diff --git a/desktop_client.py b/desktop_client.py index ee7fccf83..d1dc12fbd 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -57,6 +57,7 @@ from bookmarks import sendUndoBookmarkViaServer from delete import sendDeleteViaServer from person import getActorJson from storage import storeValue +from storage import readWholeFile def _desktopHelp() -> None: @@ -284,10 +285,9 @@ def _desktopShowBanner() -> None: bannerFilename = 'theme/' + bannerTheme + '/banner.txt' if not os.path.isfile(bannerFilename): return - with open(bannerFilename, 'r') as bannerFile: - banner = bannerFile.read() - if banner: - print(banner + '\n') + banner = readWholeFile(bannerFilename) + if banner: + print(banner + '\n') def _desktopWaitForCmd(timeout: int, debug: bool) -> str: diff --git a/follow.py b/follow.py index 9eb158491..1387c2c8a 100644 --- a/follow.py +++ b/follow.py @@ -31,6 +31,8 @@ from auth import createBasicAuthHeader from session import getJson from session import postJson from storage import storeValue +from storage import readWholeFile +from storage import readFileLines def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: @@ -49,8 +51,8 @@ def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: lastSeenDir = accountDir + '/lastseen' if not os.path.isdir(lastSeenDir): os.mkdir(lastSeenDir) - with open(followingFilename, 'r') as fp: - followingHandles = fp.readlines() + followingHandles = readFileLines(followingFilename) + if followingHandles: for handle in followingHandles: if '#' in handle: continue @@ -214,9 +216,7 @@ def isFollowerOfPerson(baseDir: str, nickname: str, domain: str, alreadyFollowing = False - followersStr = '' - with open(followersFile, 'r') as fpFollowers: - followersStr = fpFollowers.read() + followersStr = readWholeFile(followersFile) if handle in followersStr: alreadyFollowing = True @@ -556,9 +556,7 @@ def _storeFollowRequest(baseDir: str, if os.path.isfile(followersFilename): alreadyFollowing = False - followersStr = '' - with open(followersFilename, 'r') as fpFollowers: - followersStr = fpFollowers.read() + followersStr = readWholeFile(followersFilename) if approveHandle in followersStr: alreadyFollowing = True diff --git a/followingCalendar.py b/followingCalendar.py index 3b4665dbe..24cd9da12 100644 --- a/followingCalendar.py +++ b/followingCalendar.py @@ -8,6 +8,7 @@ __status__ = "Production" __module_group__ = "Calendar" import os +from storage import readWholeFile from storage import storeValue @@ -29,9 +30,8 @@ def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, if not os.path.isfile(followingFilename): return False # create a new calendar file from the following file - with open(followingFilename, 'r') as followingFile: - followingHandles = followingFile.read() - storeValue(calendarFilename, followingHandles, 'writeonly') + followingHandles = readWholeFile(followingFilename) + storeValue(calendarFilename, followingHandles, 'writeonly') return handle + '\n' in open(calendarFilename).read() @@ -66,14 +66,11 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, followingHandles = '' if os.path.isfile(calendarFilename): print('Calendar file exists') - with open(calendarFilename, 'r') as calendarFile: - followingHandles = calendarFile.read() + followingHandles = readWholeFile(calendarFilename) else: # create a new calendar file from the following file print('Creating calendar file ' + calendarFilename) - followingHandles = '' - with open(followingFilename, 'r') as followingFile: - followingHandles = followingFile.read() + followingHandles = readWholeFile(followingFilename) if add: storeValue(calendarFilename, followingHandles + handle, 'write') diff --git a/happening.py b/happening.py index 1bc8685a5..efb5c7c3f 100644 --- a/happening.py +++ b/happening.py @@ -17,6 +17,7 @@ from utils import loadJson from utils import saveJson from utils import locatePost from storage import storeValue +from storage import readWholeFile def _validUuid(testUuid: str, version=4): @@ -35,9 +36,9 @@ def _removeEventFromTimeline(eventId: str, tlEventsFilename: str) -> None: """ if eventId + '\n' not in open(tlEventsFilename).read(): return - with open(tlEventsFilename, 'r') as fp: - eventsTimeline = fp.read().replace(eventId + '\n', '') - storeValue(tlEventsFilename, eventsTimeline, 'writeonly') + eventsTimeline = readWholeFile(tlEventsFilename) + eventsTimeline = eventsTimeline.replace(eventId + '\n', '') + storeValue(tlEventsFilename, eventsTimeline, 'writeonly') def saveEventPost(baseDir: str, handle: str, postId: str, diff --git a/inbox.py b/inbox.py index e80fc9071..033f6b0d5 100644 --- a/inbox.py +++ b/inbox.py @@ -84,6 +84,7 @@ from context import hasValidContext from speaker import updateSpeaker from announce import isSelfAnnounce from storage import storeValue +from storage import readWholeFile def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -1879,10 +1880,9 @@ def _likeNotify(baseDir: str, domain: str, onionDomain: str, # was there a previous like notification? if os.path.isfile(prevLikeFile): # is it the same as the current notification ? - with open(prevLikeFile, 'r') as fp: - prevLikeStr = fp.read() - if prevLikeStr == likeStr: - return + prevLikeStr = readWholeFile(prevLikeFile) + if prevLikeStr == likeStr: + return storeValue(prevLikeFile, likeStr, 'writeonly') storeValue(likeFile, likeStr, 'writeonly') @@ -2108,8 +2108,8 @@ def _updateLastSeen(baseDir: str, handle: str, actor: str) -> None: daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days # has the value changed? if os.path.isfile(lastSeenFilename): - with open(lastSeenFilename, 'r') as lastSeenFile: - daysSinceEpochFile = lastSeenFile.read() + daysSinceEpochFile = readWholeFile(lastSeenFilename) + if daysSinceEpochFile: if int(daysSinceEpochFile) == daysSinceEpoch: # value hasn't changed, so we can save writing anything to file return diff --git a/manualapprove.py b/manualapprove.py index 7e97543b4..9c7fdd558 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -12,6 +12,7 @@ from follow import followedAccountAccepts from follow import followedAccountRejects from follow import removeFromFollowRequests from utils import loadJson +from storage import readWholeFile from storage import storeValue @@ -96,9 +97,7 @@ def manualApproveFollowRequest(session, baseDir: str, return # is the handle in the requests file? - approveFollowsStr = '' - with open(approveFollowsFilename, 'r') as fpFollowers: - approveFollowsStr = fpFollowers.read() + approveFollowsStr = readWholeFile(approveFollowsFilename) exists = False approveHandleFull = approveHandle if approveHandle in approveFollowsStr: diff --git a/media.py b/media.py index 5530f60f5..0182222ba 100644 --- a/media.py +++ b/media.py @@ -21,6 +21,7 @@ from shutil import copyfile from shutil import rmtree from shutil import move from city import spoofGeolocation +from storage import readWholeFile from storage import storeValue @@ -70,8 +71,10 @@ def _spoofMetaData(baseDir: str, nickname: str, domain: str, baseDir + '/accounts/' + nickname + '@' + domain + '/decoyseed' decoySeed = 63725 if os.path.isfile(decoySeedFilename): - with open(decoySeedFilename, 'r') as fp: - decoySeed = int(fp.read()) + decoySeedStr = readWholeFile(decoySeedFilename) + if decoySeedStr: + if decoySeedStr.isdigit(): + decoySeed = int(decoySeedStr) else: decoySeed = randint(10000, 10000000000000000) decoySeedStr = str(decoySeed) diff --git a/newsdaemon.py b/newsdaemon.py index c816417dd..8c43b0b55 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -34,6 +34,7 @@ from utils import clearFromPostCaches from utils import dangerousMarkup from inbox import storeHashTags from session import createSession +from storage import readWholeFile from storage import storeValue @@ -398,9 +399,8 @@ def _createNewsMirror(baseDir: str, domain: str, # remove the corresponding index entries if removals: - indexContent = '' - with open(mirrorIndexFilename, 'r') as indexFile: - indexContent = indexFile.read() + indexContent = readWholeFile(mirrorIndexFilename) + if indexContent: for removePostId in removals: indexContent = \ indexContent.replace(removePostId + '\n', '') diff --git a/newswire.py b/newswire.py index 51c773994..8a877d07f 100644 --- a/newswire.py +++ b/newswire.py @@ -29,6 +29,8 @@ from utils import removeHtml from blocking import isBlockedDomain from blocking import isBlockedHashtag from filters import isFiltered +from storage import readWholeFile +from storage import readFileLines def _removeCDATA(text: str) -> str: @@ -219,8 +221,8 @@ def loadHashtagCategories(baseDir: str, language: str) -> None: if not os.path.isfile(hashtagCategoriesFilename): return - with open(hashtagCategoriesFilename, 'r') as fp: - xmlStr = fp.read() + xmlStr = readWholeFile(hashtagCategoriesFilename) + if xmlStr: _xml2StrToHashtagCategories(baseDir, xmlStr, 1024, True) @@ -1014,9 +1016,7 @@ def getDictFromNewswire(session, baseDir: str, domain: str, maxPostsPerSource = 5 # add rss feeds - rssFeed = [] - with open(subscriptionsFilename, 'r') as fp: - rssFeed = fp.readlines() + rssFeed = readFileLines(subscriptionsFilename) result = {} for url in rssFeed: url = url.strip() diff --git a/person.py b/person.py index f9052ff86..eaca21988 100644 --- a/person.py +++ b/person.py @@ -52,6 +52,7 @@ from session import createSession from session import getJson from webfinger import webfingerHandle from pprint import pprint +from storage import readWholeFile from storage import storeValue @@ -1121,9 +1122,7 @@ def isPersonSnoozed(baseDir: str, nickname: str, domain: str, replaceStr = line break if replaceStr: - content = None - with open(snoozedFilename, 'r') as snoozedFile: - content = snoozedFile.read().replace(replaceStr, '') + content = readWholeFile(snoozedFilename).replace(replaceStr, '') if content: storeValue(snoozedFilename, content, 'writeonly') @@ -1168,9 +1167,7 @@ def personUnsnooze(baseDir: str, nickname: str, domain: str, replaceStr = line break if replaceStr: - content = None - with open(snoozedFilename, 'r') as snoozedFile: - content = snoozedFile.read().replace(replaceStr, '') + content = readWholeFile(snoozedFilename).replace(replaceStr, '') if content: storeValue(snoozedFilename, content, 'writeonly') diff --git a/petnames.py b/petnames.py index 52ece3c3f..7d4ffd4f5 100644 --- a/petnames.py +++ b/petnames.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from storage import readWholeFile from storage import storeValue @@ -28,8 +29,8 @@ def setPetName(baseDir: str, nickname: str, domain: str, # does this entry already exist? if os.path.isfile(petnamesFilename): - with open(petnamesFilename, 'r') as petnamesFile: - petnamesStr = petnamesFile.read() + petnamesStr = readWholeFile(petnamesFilename) + if petnamesStr: if entry in petnamesStr: return True if ' ' + handle + '\n' in petnamesStr: @@ -65,8 +66,8 @@ def getPetName(baseDir: str, nickname: str, domain: str, if not os.path.isfile(petnamesFilename): return '' - with open(petnamesFilename, 'r') as petnamesFile: - petnamesStr = petnamesFile.read() + petnamesStr = readWholeFile(petnamesFilename) + if petnamesStr: if ' ' + handle + '\n' in petnamesStr: petnamesList = petnamesStr.split('\n') for pet in petnamesList: @@ -86,8 +87,8 @@ def _getPetNameHandle(baseDir: str, nickname: str, domain: str, if not os.path.isfile(petnamesFilename): return '' - with open(petnamesFilename, 'r') as petnamesFile: - petnamesStr = petnamesFile.read() + petnamesStr = readWholeFile(petnamesFilename) + if petnamesStr: if petname + ' ' in petnamesStr: petnamesList = petnamesStr.split('\n') for pet in petnamesList: diff --git a/posts.py b/posts.py index 05291fedf..e7d0c6328 100644 --- a/posts.py +++ b/posts.py @@ -72,6 +72,7 @@ from git import convertPostToPatch from linked_data_sig import generateJsonSignature from petnames import resolvePetnames from storage import storeValue +from storage import readWholeFile def isModerator(baseDir: str, nickname: str) -> bool: @@ -3022,9 +3023,8 @@ def _addPostToTimeline(filePath: str, boxname: str, postsInBox: [], boxActor: str) -> bool: """ Reads a post from file and decides whether it is valid """ - with open(filePath, 'r') as postFile: - postStr = postFile.read() - + postStr = readWholeFile(filePath) + if postStr: if filePath.endswith('.json'): repliesFilename = filePath.replace('.json', '.replies') if os.path.isfile(repliesFilename): @@ -3715,9 +3715,7 @@ def getPublicPostDomainsBlocked(session, baseDir: str, return [] # read the blocked domains as a single string - blockedStr = '' - with open(blockingFilename, 'r') as fp: - blockedStr = fp.read() + blockedStr = readWholeFile(blockingFilename) blockedDomains = [] for domainName in postDomains: @@ -3766,8 +3764,7 @@ def checkDomains(session, baseDir: str, updateFollowerWarnings = False followerWarningStr = '' if os.path.isfile(followerWarningFilename): - with open(followerWarningFilename, 'r') as fp: - followerWarningStr = fp.read() + followerWarningStr = readWholeFile(followerWarningFilename) if singleCheck: # checks a single random non-mutual diff --git a/speaker.py b/speaker.py index a04ef3709..c1d61042d 100644 --- a/speaker.py +++ b/speaker.py @@ -22,6 +22,8 @@ from utils import loadJson from utils import saveJson from utils import isPGPEncrypted from content import htmlReplaceQuoteMarks +from storage import readFileLines +from storage import readWholeFile speakerRemoveChars = ('.\n', '. ', ',', ';', '?', '!') @@ -135,8 +137,8 @@ def _speakerPronounce(baseDir: str, sayText: str, translate: {}) -> str: ")": "," } if os.path.isfile(pronounceFilename): - with open(pronounceFilename, 'r') as fp: - pronounceList = fp.readlines() + pronounceList = readFileLines(pronounceFilename) + if pronounceList: for conversion in pronounceList: separator = None if '->' in conversion: @@ -494,8 +496,8 @@ def _postToSpeakerJson(baseDir: str, httpPrefix: str, accountsDir = baseDir + '/accounts/' + nickname + '@' + domainFull approveFollowsFilename = accountsDir + '/followrequests.txt' if os.path.isfile(approveFollowsFilename): - with open(approveFollowsFilename, 'r') as fp: - follows = fp.readlines() + follows = readFileLines(approveFollowsFilename) + if follows: if len(follows) > 0: followRequestsExist = True for i in range(len(follows)): @@ -512,8 +514,7 @@ def _postToSpeakerJson(baseDir: str, httpPrefix: str, likedBy = '' likeFilename = accountsDir + '/.newLike' if os.path.isfile(likeFilename): - with open(likeFilename, 'r') as fp: - likedBy = fp.read() + likedBy = readWholeFile(likeFilename) calendarFilename = accountsDir + '/.newCalendar' postCal = os.path.isfile(calendarFilename) shareFilename = accountsDir + '/.newShare' diff --git a/storage.py b/storage.py index 0417796f0..d78d35648 100644 --- a/storage.py +++ b/storage.py @@ -54,3 +54,19 @@ def storeValue(filename: str, lineStr: str, storeType: str) -> bool: print('WARN: Unable to prepend to ' + filename + ' ' + str(e)) return False + + +def readWholeFile(filename: str) -> str: + """Returns the entire contents of a file + """ + with open(filename, 'r') as fp: + return fp.read() + return '' + + +def readFileLines(filename: str) -> []: + """Returns a list of lines from a file + """ + with open(filename, 'r') as fp: + return fp.readlines() + return [] diff --git a/theme.py b/theme.py index 5fa2bac01..600faadbf 100644 --- a/theme.py +++ b/theme.py @@ -17,6 +17,7 @@ from shutil import unpack_archive from shutil import rmtree from content import dangerousCSS from storage import storeValue +from storage import readWholeFile def importTheme(baseDir: str, filename: str) -> bool: @@ -35,9 +36,9 @@ def importTheme(baseDir: str, filename: str) -> bool: print('WARN: ' + themeFile + ' missing from imported theme') return False - newThemeName = None - with open(tempThemeDir + '/name.txt', 'r') as fp: - newThemeName = fp.read().replace('\n', '').replace('\r', '') + newThemeName = readWholeFile(tempThemeDir + '/name.txt') + if newThemeName: + newThemeName = newThemeName.replace('\n', '').replace('\r', '') if len(newThemeName) > 20: print('WARN: Imported theme name is too long') return False @@ -327,8 +328,8 @@ def _setThemeFromDict(baseDir: str, name: str, if not os.path.isfile(templateFilename): continue - with open(templateFilename, 'r') as cssfile: - css = cssfile.read() + css = readWholeFile(templateFilename) + if css: for paramName, paramValue in themeParams.items(): if paramName == 'newswire-publish-icon': if paramValue.lower() == 'true': @@ -385,8 +386,8 @@ def _setBackgroundFormat(baseDir: str, name: str, cssFilename = baseDir + '/' + backgroundType + '.css' if not os.path.isfile(cssFilename): return - with open(cssFilename, 'r') as cssfile: - css = cssfile.read() + css = readWholeFile(cssFilename) + if css: css = css.replace('background.jpg', 'background.' + extension) storeValue(cssFilename, css, 'writeonly') @@ -399,8 +400,8 @@ def enableGrayscale(baseDir: str) -> None: templateFilename = baseDir + '/' + filename if not os.path.isfile(templateFilename): continue - with open(templateFilename, 'r') as cssfile: - css = cssfile.read() + css = readWholeFile(templateFilename) + if css: if 'grayscale' not in css: css = \ css.replace('body, html {', @@ -420,8 +421,8 @@ def disableGrayscale(baseDir: str) -> None: templateFilename = baseDir + '/' + filename if not os.path.isfile(templateFilename): continue - with open(templateFilename, 'r') as cssfile: - css = cssfile.read() + css = readWholeFile(templateFilename) + if css: if 'grayscale' in css: css = \ css.replace('\n filter: grayscale(100%);', '') @@ -456,8 +457,8 @@ def _setCustomFont(baseDir: str): templateFilename = baseDir + '/' + filename if not os.path.isfile(templateFilename): continue - with open(templateFilename, 'r') as cssfile: - css = cssfile.read() + css = readWholeFile(templateFilename) + if css: css = \ setCSSparam(css, "*src", "url('./fonts/custom." + @@ -540,10 +541,9 @@ def getTextModeBanner(baseDir: str) -> str: """ textModeBannerFilename = baseDir + '/accounts/banner.txt' if os.path.isfile(textModeBannerFilename): - with open(textModeBannerFilename, 'r') as fp: - bannerStr = fp.read() - if bannerStr: - return bannerStr.replace('\n', '
') + bannerStr = readWholeFile(textModeBannerFilename) + if bannerStr: + return bannerStr.replace('\n', '
') return None @@ -554,10 +554,9 @@ def getTextModeLogo(baseDir: str) -> str: if not os.path.isfile(textModeLogoFilename): textModeLogoFilename = baseDir + '/img/logo.txt' - with open(textModeLogoFilename, 'r') as fp: - logoStr = fp.read() - if logoStr: - return logoStr.replace('\n', '
') + logoStr = readWholeFile(textModeLogoFilename) + if logoStr: + return logoStr.replace('\n', '
') return None diff --git a/utils.py b/utils.py index 85d24364e..41abdba78 100644 --- a/utils.py +++ b/utils.py @@ -19,6 +19,7 @@ from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from storage import storeValue +from storage import readWholeFile # posts containing these strings will always get screened out, # both incoming and outgoing. @@ -163,8 +164,8 @@ def isDormant(baseDir: str, nickname: str, domain: str, actor: str, if not os.path.isfile(lastSeenFilename): return False - with open(lastSeenFilename, 'r') as lastSeenFile: - daysSinceEpochStr = lastSeenFile.read() + daysSinceEpochStr = readWholeFile(lastSeenFilename) + if daysSinceEpochStr: daysSinceEpoch = int(daysSinceEpochStr) currTime = datetime.datetime.utcnow() currDaysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days @@ -506,8 +507,8 @@ def loadJson(filename: str, delaySec=2, maxTries=5) -> {}: tries = 0 while tries < maxTries: try: - with open(filename, 'r') as fp: - data = fp.read() + data = readWholeFile(filename) + if data: jsonObject = json.loads(data) break except BaseException: @@ -527,14 +528,13 @@ def loadJsonOnionify(filename: str, domain: str, onionDomain: str, tries = 0 while tries < 5: try: - with open(filename, 'r') as fp: - data = fp.read() - if data: - data = data.replace(domain, onionDomain) - data = data.replace('https:', 'http:') - print('*****data: ' + data) - jsonObject = json.loads(data) - break + data = readWholeFile(filename) + if data: + data = data.replace(domain, onionDomain) + data = data.replace('https:', 'http:') + print('*****data: ' + data) + jsonObject = json.loads(data) + break except BaseException: print('WARN: loadJson exception') if delaySec > 0: @@ -942,14 +942,13 @@ def _setDefaultPetName(baseDir: str, nickname: str, domain: str, storeValue(petnamesFilename, petnameLookupEntry, 'writeonly') return - with open(petnamesFilename, 'r') as petnamesFile: - petnamesStr = petnamesFile.read() - if petnamesStr: - petnamesList = petnamesStr.split('\n') - for pet in petnamesList: - if pet.startswith(followNickname + ' '): - # petname already exists - return + petnamesStr = readWholeFile(petnamesFilename) + if petnamesStr: + petnamesList = petnamesStr.split('\n') + for pet in petnamesList: + if pet.startswith(followNickname + ' '): + # petname already exists + return # petname doesn't already exist storeValue(petnamesFilename, petnameLookupEntry, 'append') @@ -1094,13 +1093,12 @@ def locateNewsArrival(baseDir: str, domain: str, accountDir = baseDir + '/accounts/news@' + domain + '/' postFilename = accountDir + 'outbox/' + postUrl if os.path.isfile(postFilename): - with open(postFilename, 'r') as arrivalFile: - arrival = arrivalFile.read() - if arrival: - arrivalDate = \ - datetime.datetime.strptime(arrival, - "%Y-%m-%dT%H:%M:%SZ") - return arrivalDate + arrival = readWholeFile(postFilename) + if arrival: + arrivalDate = \ + datetime.datetime.strptime(arrival, + "%Y-%m-%dT%H:%M:%SZ") + return arrivalDate return None @@ -1488,8 +1486,8 @@ def noOfActiveAccountsMonthly(baseDir: str, months: int) -> bool: lastUsedFilename = \ baseDir + '/accounts/' + account + '/.lastUsed' if os.path.isfile(lastUsedFilename): - with open(lastUsedFilename, 'r') as lastUsedFile: - lastUsed = lastUsedFile.read() + lastUsed = readWholeFile(lastUsedFilename) + if lastUsed: if lastUsed.isdigit(): timeDiff = (currTime - int(lastUsed)) if timeDiff < monthSeconds: @@ -1645,8 +1643,8 @@ def getCSS(baseDir: str, cssFilename: str, cssCache: {}) -> str: # file hasn't changed, so return the version in the cache return cssCache[cssFilename][1] - with open(cssFilename, 'r') as fpCSS: - css = fpCSS.read() + css = readWholeFile(cssFilename) + if css: if cssCache.get(cssFilename): # alter the cache contents cssCache[cssFilename][0] = lastModified @@ -1756,9 +1754,8 @@ def _searchVirtualBoxPosts(baseDir: str, nickname: str, domain: str, postFilename = path + '/' + postFilename.strip() if not os.path.isfile(postFilename): continue - with open(postFilename, 'r') as postFile: - data = postFile.read().lower() - + data = readWholeFile(postFilename).lower() + if data: notFound = False for keyword in searchWords: if keyword not in data: @@ -1799,9 +1796,8 @@ def searchBoxPosts(baseDir: str, nickname: str, domain: str, for root, dirs, fnames in os.walk(path): for fname in fnames: filePath = os.path.join(root, fname) - with open(filePath, 'r') as postFile: - data = postFile.read().lower() - + data = readWholeFile(filePath).lower() + if data: notFound = False for keyword in searchWords: if keyword not in data: diff --git a/webapp_about.py b/webapp_about.py index 233d052f7..a9f468007 100644 --- a/webapp_about.py +++ b/webapp_about.py @@ -13,6 +13,7 @@ from utils import getConfigParam from webapp_utils import htmlHeaderWithWebsiteMarkup from webapp_utils import htmlFooter from webapp_utils import markdownToHtml +from storage import readWholeFile def htmlAbout(cssCache: {}, baseDir: str, httpPrefix: str, @@ -32,8 +33,9 @@ def htmlAbout(cssCache: {}, baseDir: str, httpPrefix: str, aboutText = 'Information about this instance goes here.' if os.path.isfile(baseDir + '/accounts/about.md'): - with open(baseDir + '/accounts/about.md', 'r') as aboutFile: - aboutText = markdownToHtml(aboutFile.read()) + aboutText = readWholeFile(baseDir + '/accounts/about.md') + if aboutText: + aboutText = markdownToHtml(aboutText) aboutForm = '' cssFilename = baseDir + '/epicyon-profile.css' diff --git a/webapp_column_left.py b/webapp_column_left.py index 5428650f0..5fc2cfe82 100644 --- a/webapp_column_left.py +++ b/webapp_column_left.py @@ -18,6 +18,7 @@ from webapp_utils import headerButtonsFrontScreen from webapp_utils import htmlHeaderWithExternalStyle from webapp_utils import htmlFooter from webapp_utils import getBannerFile +from storage import readWholeFile def _linksExist(baseDir: str) -> bool: @@ -403,8 +404,7 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, linksFilename = baseDir + '/accounts/links.txt' linksStr = '' if os.path.isfile(linksFilename): - with open(linksFilename, 'r') as fp: - linksStr = fp.read() + linksStr = readWholeFile(linksFilename) editLinksForm += \ '
' @@ -426,8 +426,7 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, aboutFilename = baseDir + '/accounts/about.md' aboutStr = '' if os.path.isfile(aboutFilename): - with open(aboutFilename, 'r') as fp: - aboutStr = fp.read() + aboutStr = readWholeFile(aboutFilename) editLinksForm += \ '
' @@ -445,8 +444,7 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, TOSFilename = baseDir + '/accounts/tos.md' TOSStr = '' if os.path.isfile(TOSFilename): - with open(TOSFilename, 'r') as fp: - TOSStr = fp.read() + TOSStr = readWholeFile(TOSFilename) editLinksForm += \ '
' diff --git a/webapp_column_right.py b/webapp_column_right.py index 0d194c1b0..6a5664630 100644 --- a/webapp_column_right.py +++ b/webapp_column_right.py @@ -24,6 +24,7 @@ from webapp_utils import htmlFooter from webapp_utils import getBannerFile from webapp_utils import htmlPostSeparator from webapp_utils import headerButtonsFrontScreen +from storage import readWholeFile def _votesIndicator(totalVotes: int, positiveVoting: bool) -> str: @@ -576,8 +577,7 @@ def htmlEditNewswire(cssCache: {}, translate: {}, baseDir: str, path: str, newswireFilename = baseDir + '/accounts/newswire.txt' newswireStr = '' if os.path.isfile(newswireFilename): - with open(newswireFilename, 'r') as fp: - newswireStr = fp.read() + newswireStr = readWholeFile(newswireFilename) editNewswireForm += \ '
' @@ -595,8 +595,7 @@ def htmlEditNewswire(cssCache: {}, translate: {}, baseDir: str, path: str, filterFilename = \ baseDir + '/accounts/news@' + domain + '/filters.txt' if os.path.isfile(filterFilename): - with open(filterFilename, 'r') as filterfile: - filterStr = filterfile.read() + filterStr = readWholeFile(filterFilename) editNewswireForm += \ '