From 9c18e7042e76c6a1282b4f0ab86c58eeeaf197d4 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 22:02:03 +0100 Subject: [PATCH] File storage functions --- auth.py | 26 +++---- blocking.py | 6 +- bookmarks.py | 11 +-- categories.py | 7 +- daemon.py | 158 +++++++++++++++---------------------------- desktop_client.py | 6 +- epicyon.py | 9 +-- follow.py | 14 ++-- followingCalendar.py | 13 ++-- git.py | 7 +- happening.py | 31 ++++----- inbox.py | 63 ++++------------- media.py | 14 ++-- migrate.py | 9 +-- newsdaemon.py | 24 ++----- person.py | 30 +++----- petnames.py | 10 ++- posts.py | 50 +++----------- question.py | 21 +++--- shares.py | 11 ++- storage.py | 56 +++++++++++++++ tests.py | 28 ++++---- theme.py | 22 +++--- utils.py | 36 ++++------ webapp_post.py | 15 ++-- webapp_utils.py | 4 +- webapp_welcome.py | 6 +- 27 files changed, 267 insertions(+), 420 deletions(-) create mode 100644 storage.py diff --git a/auth.py b/auth.py index 87b94e5e2..c62dc8236 100644 --- a/auth.py +++ b/auth.py @@ -15,6 +15,7 @@ import secrets import datetime from utils import isSystemAccount from utils import hasUsersPath +from storage import storeValue def _hashPassword(password: str) -> str: @@ -175,8 +176,7 @@ def storeBasicCredentials(baseDir: str, nickname: str, password: str) -> bool: with open(passwordFile, 'a+') as passfile: passfile.write(storeStr + '\n') else: - with open(passwordFile, 'w+') as passfile: - passfile.write(storeStr + '\n') + storeValue(passwordFile, storeStr, 'write') return True @@ -240,18 +240,14 @@ def recordLoginFailure(baseDir: str, ipAddress: str, return failureLog = baseDir + '/accounts/loginfailures.log' - writeType = 'a+' + writeType = 'append' if not os.path.isfile(failureLog): - writeType = 'w+' + writeType = 'writeonly' currTime = datetime.datetime.utcnow() - try: - with open(failureLog, writeType) as fp: - # here we use a similar format to an ssh log, so that - # systems such as fail2ban can parse it - fp.write(currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + - 'ip-127-0-0-1 sshd[20710]: ' + - 'Disconnecting invalid user epicyon ' + - ipAddress + ' port 443: ' + - 'Too many authentication failures [preauth]\n') - except BaseException: - pass + logLineStr = \ + currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + \ + 'ip-127-0-0-1 sshd[20710]: ' + \ + 'Disconnecting invalid user epicyon ' + \ + ipAddress + ' port 443: ' + \ + 'Too many authentication failures [preauth]\n' + storeValue(failureLog, logLineStr, writeType) diff --git a/blocking.py b/blocking.py index 86a6c052d..a90a63953 100644 --- a/blocking.py +++ b/blocking.py @@ -25,6 +25,7 @@ from utils import locatePost from utils import evilIncarnate from utils import getDomainFromActor from utils import getNicknameFromActor +from storage import storeValue def addGlobalBlock(baseDir: str, @@ -493,10 +494,7 @@ def mutePost(baseDir: str, nickname: str, domain: str, port: int, if os.path.isfile(cachedPostFilename): os.remove(cachedPostFilename) - muteFile = open(postFilename + '.muted', 'w+') - if muteFile: - muteFile.write('\n') - muteFile.close() + if storeValue(postFilename + '.muted', '\n', 'writeonly'): print('MUTE: ' + postFilename + '.muted file added') # if the post is in the recent posts cache then mark it as muted diff --git a/bookmarks.py b/bookmarks.py index 58a772b3d..3858d22e7 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -24,6 +24,7 @@ from utils import loadJson from utils import saveJson from posts import getPersonBox from session import postJson +from storage import storeValue def undoBookmarksCollectionEntry(recentPostsCache: {}, @@ -61,10 +62,7 @@ def undoBookmarksCollectionEntry(recentPostsCache: {}, indexStr = '' with open(bookmarksIndexFilename, 'r') as indexFile: indexStr = indexFile.read().replace(bookmarkIndex + '\n', '') - bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') - if bookmarksIndexFile: - bookmarksIndexFile.write(indexStr) - bookmarksIndexFile.close() + storeValue(bookmarksIndexFilename, indexStr, 'writeonly') if not postJsonObject.get('type'): return @@ -219,10 +217,7 @@ def updateBookmarksCollection(recentPostsCache: {}, print('WARN: Failed to write entry to bookmarks index ' + bookmarksIndexFilename + ' ' + str(e)) else: - bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') - if bookmarksIndexFile: - bookmarksIndexFile.write(bookmarkIndex + '\n') - bookmarksIndexFile.close() + storeValue(bookmarksIndexFilename, bookmarkIndex, 'write') def bookmark(recentPostsCache: {}, diff --git a/categories.py b/categories.py index 70d079a78..67c65ba15 100644 --- a/categories.py +++ b/categories.py @@ -9,6 +9,7 @@ __module_group__ = "RSS Feeds" import os import datetime +from storage import storeValue def getHashtagCategory(baseDir: str, hashtag: str) -> str: @@ -106,8 +107,7 @@ def _updateHashtagCategories(baseDir: str) -> None: categoryListStr += categoryStr + '\n' # save a list of available categories for quick lookup - with open(categoryListFilename, 'w+') as fp: - fp.write(categoryListStr) + storeValue(categoryListFilename, categoryListStr, 'writeonly') def _validHashtagCategory(category: str) -> bool: @@ -153,8 +153,7 @@ def setHashtagCategory(baseDir: str, hashtag: str, category: str, # don't overwrite any existing categories if os.path.isfile(categoryFilename): return False - with open(categoryFilename, 'w+') as fp: - fp.write(category) + if storeValue(categoryFilename, category, 'writeonly'): _updateHashtagCategories(baseDir) return True diff --git a/daemon.py b/daemon.py index 223eace39..db2cffa52 100644 --- a/daemon.py +++ b/daemon.py @@ -300,6 +300,7 @@ from context import hasValidContext from speaker import getSSMLbox from city import getSpoofedCity import os +from storage import storeValue # maximum number of posts to list in outbox feed @@ -674,11 +675,7 @@ class PubServer(BaseHTTPRequestHandler): pass if not etag: etag = sha1(data).hexdigest() # nosec - try: - with open(mediaFilename + '.etag', 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaFilename + '.etag', etag, 'writeonly') if etag: self.send_header('ETag', etag) self.end_headers() @@ -1545,12 +1542,7 @@ class PubServer(BaseHTTPRequestHandler): print('WARN: Unable to read salt for ' + loginNickname + ' ' + str(e)) else: - try: - with open(saltFilename, 'w+') as fp: - fp.write(salt) - except Exception as e: - print('WARN: Unable to save salt for ' + - loginNickname + ' ' + str(e)) + storeValue(saltFilename, salt, 'writeonly') tokenText = loginNickname + loginPassword + salt token = sha256(tokenText.encode('utf-8')).hexdigest() @@ -1559,12 +1551,7 @@ class PubServer(BaseHTTPRequestHandler): tokenFilename = \ baseDir+'/accounts/' + \ loginHandle + '/.token' - try: - with open(tokenFilename, 'w+') as fp: - fp.write(token) - except Exception as e: - print('WARN: Unable to save token for ' + - loginNickname + ' ' + str(e)) + storeValue(tokenFilename, token, 'writeonly') personUpgradeActor(baseDir, None, loginHandle, baseDir + '/accounts/' + @@ -2104,10 +2091,8 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - noNewswireFile = open(newswireBlockedFilename, "w+") - if noNewswireFile: - noNewswireFile.write('\n') - noNewswireFile.close() + if storeValue(newswireBlockedFilename, + '\n', 'writeonly'): refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2140,10 +2125,8 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - noFeaturesFile = open(featuresBlockedFilename, "w+") - if noFeaturesFile: - noFeaturesFile.write('\n') - noFeaturesFile.close() + if storeValue(featuresBlockedFilename, + '\n', 'writeonly'): refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2175,10 +2158,7 @@ class PubServer(BaseHTTPRequestHandler): os.remove(newswireModFilename) else: if os.path.isdir(accountDir): - modNewswireFile = open(newswireModFilename, "w+") - if modNewswireFile: - modNewswireFile.write('\n') - modNewswireFile.close() + storeValue(newswireModFilename, '\n', 'writeonly') usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ '?page=' + str(pageNumber) @@ -3459,10 +3439,7 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('editedLinks'): linksStr = fields['editedLinks'] - linksFile = open(linksFilename, "w+") - if linksFile: - linksFile.write(linksStr) - linksFile.close() + storeValue(linksFilename, linksStr, 'writeonly') else: if os.path.isfile(linksFilename): os.remove(linksFilename) @@ -3474,10 +3451,7 @@ class PubServer(BaseHTTPRequestHandler): aboutStr = fields['editedAbout'] if not dangerousMarkup(aboutStr, allowLocalNetworkAccess): - aboutFile = open(aboutFilename, "w+") - if aboutFile: - aboutFile.write(aboutStr) - aboutFile.close() + storeValue(aboutFilename, aboutStr, 'writeonly') else: if os.path.isfile(aboutFilename): os.remove(aboutFilename) @@ -3486,10 +3460,7 @@ class PubServer(BaseHTTPRequestHandler): TOSStr = fields['editedTOS'] if not dangerousMarkup(TOSStr, allowLocalNetworkAccess): - TOSFile = open(TOSFilename, "w+") - if TOSFile: - TOSFile.write(TOSStr) - TOSFile.close() + storeValue(TOSFilename, TOSStr, 'writeonly') else: if os.path.isfile(TOSFilename): os.remove(TOSFilename) @@ -3664,10 +3635,7 @@ class PubServer(BaseHTTPRequestHandler): extractTextFieldsInPOST(postBytes, boundary, debug) if fields.get('editedNewswire'): newswireStr = fields['editedNewswire'] - newswireFile = open(newswireFilename, "w+") - if newswireFile: - newswireFile.write(newswireStr) - newswireFile.close() + storeValue(newswireFilename, newswireStr, 'writeonly') else: if os.path.isfile(newswireFilename): os.remove(newswireFilename) @@ -3677,8 +3645,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ 'news@' + domain + '/filters.txt' if fields.get('filteredWordsNewswire'): - with open(filterNewswireFilename, 'w+') as filterfile: - filterfile.write(fields['filteredWordsNewswire']) + storeValue(filterNewswireFilename, + fields['filteredWordsNewswire'], 'writeonly') else: if os.path.isfile(filterNewswireFilename): os.remove(filterNewswireFilename) @@ -3687,8 +3655,8 @@ class PubServer(BaseHTTPRequestHandler): hashtagRulesFilename = \ baseDir + '/accounts/hashtagrules.txt' if fields.get('hashtagRulesList'): - with open(hashtagRulesFilename, 'w+') as rulesfile: - rulesfile.write(fields['hashtagRulesList']) + storeValue(hashtagRulesFilename, + fields['hashtagRulesList'], 'writeonly') else: if os.path.isfile(hashtagRulesFilename): os.remove(hashtagRulesFilename) @@ -3698,10 +3666,8 @@ class PubServer(BaseHTTPRequestHandler): newswireTrusted = fields['trustedNewswire'] if not newswireTrusted.endswith('\n'): newswireTrusted += '\n' - trustFile = open(newswireTrustedFilename, "w+") - if trustFile: - trustFile.write(newswireTrusted) - trustFile.close() + storeValue(newswireTrustedFilename, + newswireTrusted, 'writeonly') else: if os.path.isfile(newswireTrustedFilename): os.remove(newswireTrustedFilename) @@ -3787,10 +3753,8 @@ class PubServer(BaseHTTPRequestHandler): citationsStr += citationDate + '\n' # save citations dates, so that they can be added when # reloading the newblog screen - citationsFile = open(citationsFilename, "w+") - if citationsFile: - citationsFile.write(citationsStr) - citationsFile.close() + storeValue(citationsFilename, + citationsStr, 'writeonly') # redirect back to the default timeline self._redirect_headers(actorStr + '/newblog', @@ -4226,8 +4190,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('cityDropdown'): cityFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/city.txt' - with open(cityFilename, 'w+') as fp: - fp.write(fields['cityDropdown']) + storeValue(cityFilename, + fields['cityDropdown'], 'writeonly') # change displayed name if fields.get('displayNickname'): @@ -5000,16 +4964,15 @@ class PubServer(BaseHTTPRequestHandler): if onFinalWelcomeScreen: # initial default setting created via # the welcome screen - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, '\n', 'writeonly') actorChanged = True else: followDMsActive = False if fields.get('followDMs'): if fields['followDMs'] == 'on': followDMsActive = True - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, + '\n', 'writeonly') if not followDMsActive: if os.path.isfile(followDMsFilename): os.remove(followDMsFilename) @@ -5023,9 +4986,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('removeTwitter'): if fields['removeTwitter'] == 'on': removeTwitterActive = True - with open(removeTwitterFilename, - 'w+') as rFile: - rFile.write('\n') + storeValue(removeTwitterFilename, + '\n', 'writeonly') if not removeTwitterActive: if os.path.isfile(removeTwitterFilename): os.remove(removeTwitterFilename) @@ -5043,8 +5005,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('hideLikeButton'): if fields['hideLikeButton'] == 'on': hideLikeButtonActive = True - with open(hideLikeButtonFile, 'w+') as rFile: - rFile.write('\n') + storeValue(hideLikeButtonFile, + '\n', 'writeonly') # remove notify likes selection if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5055,8 +5017,8 @@ class PubServer(BaseHTTPRequestHandler): # notify about new Likes if onFinalWelcomeScreen: # default setting from welcome screen - with open(notifyLikesFilename, 'w+') as rFile: - rFile.write('\n') + storeValue(notifyLikesFilename, + '\n', 'writeonly') actorChanged = True else: notifyLikesActive = False @@ -5064,8 +5026,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['notifyLikes'] == 'on' and \ not hideLikeButtonActive: notifyLikesActive = True - with open(notifyLikesFilename, 'w+') as rFile: - rFile.write('\n') + storeValue(notifyLikesFilename, + '\n', 'writeonly') if not notifyLikesActive: if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5108,8 +5070,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/filters.txt' if fields.get('filteredWords'): - with open(filterFilename, 'w+') as filterfile: - filterfile.write(fields['filteredWords']) + storeValue(filterFilename, + fields['filteredWords'], 'writeonly') else: if os.path.isfile(filterFilename): os.remove(filterFilename) @@ -5120,8 +5082,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/replacewords.txt' if fields.get('switchWords'): - with open(switchFilename, 'w+') as switchfile: - switchfile.write(fields['switchWords']) + storeValue(switchFilename, + fields['switchWords'], 'writeonly') else: if os.path.isfile(switchFilename): os.remove(switchFilename) @@ -5132,8 +5094,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autotags.txt' if fields.get('autoTags'): - with open(autoTagsFilename, 'w+') as autoTagsFile: - autoTagsFile.write(fields['autoTags']) + storeValue(autoTagsFilename, + fields['autoTags'], 'writeonly') else: if os.path.isfile(autoTagsFilename): os.remove(autoTagsFilename) @@ -5144,8 +5106,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autocw.txt' if fields.get('autoCW'): - with open(autoCWFilename, 'w+') as autoCWFile: - autoCWFile.write(fields['autoCW']) + storeValue(autoCWFilename, + fields['autoCW'], 'writeonly') else: if os.path.isfile(autoCWFilename): os.remove(autoCWFilename) @@ -5156,8 +5118,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/blocking.txt' if fields.get('blocked'): - with open(blockedFilename, 'w+') as blockedfile: - blockedfile.write(fields['blocked']) + storeValue(blockedFilename, + fields['blocked'], 'writeonly') else: if os.path.isfile(blockedFilename): os.remove(blockedFilename) @@ -5169,8 +5131,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/dmAllowedinstances.txt' if fields.get('dmAllowedInstances'): - with open(dmAllowedInstancesFilename, 'w+') as aFile: - aFile.write(fields['dmAllowedInstances']) + storeValue(dmAllowedInstancesFilename, + fields['dmAllowedInstances'], 'writeonly') else: if os.path.isfile(dmAllowedInstancesFilename): os.remove(dmAllowedInstancesFilename) @@ -5181,8 +5143,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/allowedinstances.txt' if fields.get('allowedInstances'): - with open(allowedInstancesFilename, 'w+') as aFile: - aFile.write(fields['allowedInstances']) + storeValue(allowedInstancesFilename, + fields['allowedInstances'], 'writeonly') else: if os.path.isfile(allowedInstancesFilename): os.remove(allowedInstancesFilename) @@ -5197,8 +5159,8 @@ class PubServer(BaseHTTPRequestHandler): path.startswith('/users/' + adminNickname + '/'): self.server.peertubeInstances.clear() - with open(peertubeInstancesFile, 'w+') as aFile: - aFile.write(fields['ptInstances']) + storeValue(peertubeInstancesFile, + fields['ptInstances'], 'writeonly') ptInstancesList = \ fields['ptInstances'].split('\n') if ptInstancesList: @@ -5220,8 +5182,9 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/gitprojects.txt' if fields.get('gitProjects'): - with open(gitProjectsFilename, 'w+') as aFile: - aFile.write(fields['gitProjects'].lower()) + projectsStr = fields['gitProjects'].lower() + storeValue(gitProjectsFilename, + projectsStr, 'writeonly') else: if os.path.isfile(gitProjectsFilename): os.remove(gitProjectsFilename) @@ -13157,11 +13120,7 @@ class PubServer(BaseHTTPRequestHandler): with open(mediaFilename, 'rb') as avFile: mediaBinary = avFile.read() etag = sha1(mediaBinary).hexdigest() # nosec - try: - with open(mediaTagFilename, 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaTagFilename, etag, 'writeonly') mediaFileType = mediaFileMimeType(checkPath) self._set_headers_head(mediaFileType, fileLength, @@ -13326,13 +13285,8 @@ class PubServer(BaseHTTPRequestHandler): lastUsedFilename = \ self.server.baseDir + '/accounts/' + \ nickname + '@' + self.server.domain + '/.lastUsed' - try: - lastUsedFile = open(lastUsedFilename, 'w+') - if lastUsedFile: - lastUsedFile.write(str(int(time.time()))) - lastUsedFile.close() - except BaseException: - pass + lastUsedStr = str(int(time.time())) + storeValue(lastUsedFilename, lastUsedStr, 'writeonly') mentionsStr = '' if fields.get('mentions'): diff --git a/desktop_client.py b/desktop_client.py index ce47c52aa..ee7fccf83 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -56,6 +56,7 @@ from bookmarks import sendBookmarkViaServer from bookmarks import sendUndoBookmarkViaServer from delete import sendDeleteViaServer from person import getActorJson +from storage import storeValue def _desktopHelp() -> None: @@ -175,10 +176,7 @@ def _markPostAsRead(actor: str, postId: str, postCategory: str) -> None: except Exception as e: print('WARN: Failed to mark post as read' + str(e)) else: - readFile = open(readPostsFilename, 'w+') - if readFile: - readFile.write(postId + '\n') - readFile.close() + storeValue(readPostsFilename, postId, 'write') def _hasReadPost(actor: str, postId: str, postCategory: str) -> bool: diff --git a/epicyon.py b/epicyon.py index a36e4af93..d4f839135 100644 --- a/epicyon.py +++ b/epicyon.py @@ -88,6 +88,7 @@ from announce import sendAnnounceViaServer from socnet import instancesGraph from migrate import migrateAccounts from desktop_client import runDesktopClient +from storage import storeValue def str2bool(v) -> bool: @@ -759,12 +760,8 @@ if args.socnet: proxyType, args.port, httpPrefix, debug, __version__) - try: - with open('socnet.dot', 'w+') as fp: - fp.write(dotGraph) - print('Saved to socnet.dot') - except BaseException: - pass + if storeValue('socnet.dot', dotGraph, 'writeonly'): + print('Saved to socnet.dot') sys.exit() if args.postsraw: diff --git a/follow.py b/follow.py index f7a48ed10..0d447cf8f 100644 --- a/follow.py +++ b/follow.py @@ -30,6 +30,7 @@ from webfinger import webfingerHandle from auth import createBasicAuthHeader from session import getJson from session import postJson +from storage import storeValue def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: @@ -64,8 +65,7 @@ def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: lastSeenDir + '/' + actor.replace('/', '#') + '.txt' print('lastSeenFilename: ' + lastSeenFilename) if not os.path.isfile(lastSeenFilename): - with open(lastSeenFilename, 'w+') as fp: - fp.write(str(100)) + storeValue(lastSeenFilename, '100', 'writeonly') break @@ -279,8 +279,7 @@ def unfollowAccount(baseDir: str, nickname: str, domain: str, with open(unfollowedFilename, "a+") as f: f.write(handleToUnfollow + '\n') else: - with open(unfollowedFilename, "w+") as f: - f.write(handleToUnfollow + '\n') + storeValue(unfollowedFilename, handleToUnfollow, 'write') return True @@ -607,8 +606,7 @@ def _storeFollowRequest(baseDir: str, print('DEBUG: ' + approveHandleStored + ' is already awaiting approval') else: - with open(approveFollowsFilename, "w+") as fp: - fp.write(approveHandleStored + '\n') + storeValue(approveFollowsFilename, approveHandleStored, 'write') # store the follow request in its own directory # We don't rely upon the inbox because items in there could expire @@ -765,9 +763,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, 'Failed to write entry to followers file ' + str(e)) else: - followersFile = open(followersFilename, "w+") - followersFile.write(approveHandle + '\n') - followersFile.close() + storeValue(followersFilename, approveHandle, 'write') print('Beginning follow accept') return followedAccountAccepts(session, baseDir, httpPrefix, diff --git a/followingCalendar.py b/followingCalendar.py index 650687e14..3b4665dbe 100644 --- a/followingCalendar.py +++ b/followingCalendar.py @@ -8,6 +8,7 @@ __status__ = "Production" __module_group__ = "Calendar" import os +from storage import storeValue def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, @@ -30,8 +31,7 @@ def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, # create a new calendar file from the following file with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') return handle + '\n' in open(calendarFilename).read() @@ -75,8 +75,7 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() if add: - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles + handle + '\n') + storeValue(calendarFilename, followingHandles + handle, 'write') # already in the calendar file? if handle + '\n' in followingHandles: @@ -86,16 +85,14 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, return # remove from calendar file followingHandles = followingHandles.replace(handle + '\n', '') - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') else: print(handle + ' not in followingCalendar.txt') # not already in the calendar file if add: # append to the list of handles followingHandles += handle + '\n' - with open(calendarFilename, 'w+') as fp: - fp.write(followingHandles) + storeValue(calendarFilename, followingHandles, 'writeonly') def addPersonToCalendar(baseDir: str, nickname: str, domain: str, diff --git a/git.py b/git.py index 71fdc1442..6f66d9c3e 100644 --- a/git.py +++ b/git.py @@ -9,6 +9,7 @@ __module_group__ = "ActivityPub" import os import html +from storage import storeValue def _gitFormatContent(content: str) -> str: @@ -211,12 +212,10 @@ def receiveGitPatch(baseDir: str, nickname: str, domain: str, return False patchStr = \ _gitAddFromHandle(patchStr, '@' + fromNickname + '@' + fromDomain) - with open(patchFilename, 'w+') as patchFile: - patchFile.write(patchStr) + if storeValue(patchFilename, patchStr, 'writeonly'): patchNotifyFilename = \ baseDir + '/accounts/' + \ nickname + '@' + domain + '/.newPatchContent' - with open(patchNotifyFilename, 'w+') as patchFile: - patchFile.write(patchStr) + if storeValue(patchNotifyFilename, patchStr, 'writeonly'): return True return False diff --git a/happening.py b/happening.py index f1443164a..730b45162 100644 --- a/happening.py +++ b/happening.py @@ -16,6 +16,7 @@ from utils import isPublicPost from utils import loadJson from utils import saveJson from utils import locatePost +from storage import storeValue def _validUuid(testUuid: str, version=4): @@ -36,12 +37,7 @@ def _removeEventFromTimeline(eventId: str, tlEventsFilename: str) -> None: return with open(tlEventsFilename, 'r') as fp: eventsTimeline = fp.read().replace(eventId + '\n', '') - try: - with open(tlEventsFilename, 'w+') as fp2: - fp2.write(eventsTimeline) - except BaseException: - print('ERROR: unable to save events timeline') - pass + storeValue(tlEventsFilename, eventsTimeline, 'writeonly') def saveEventPost(baseDir: str, handle: str, postId: str, @@ -105,9 +101,7 @@ def saveEventPost(baseDir: str, handle: str, postId: str, tlEventsFilename + ' ' + str(e)) return False else: - tlEventsFile = open(tlEventsFilename, 'w+') - tlEventsFile.write(eventId + '\n') - tlEventsFile.close() + storeValue(tlEventsFilename, eventId, 'write') # create a directory for the calendar year if not os.path.isdir(calendarPath + '/' + str(eventYear)): @@ -134,17 +128,16 @@ def saveEventPost(baseDir: str, handle: str, postId: str, # a new event has been added calendarNotificationFilename = \ baseDir + '/accounts/' + handle + '/.newCalendar' - calendarNotificationFile = \ - open(calendarNotificationFilename, 'w+') - if not calendarNotificationFile: + calEventStr = \ + '/calendar?year=' + \ + str(eventYear) + \ + '?month=' + \ + str(eventMonthNumber) + \ + '?day=' + \ + str(eventDayOfMonth) + if not storeValue(calendarNotificationFilename, + calEventStr, 'write'): return False - calendarNotificationFile.write('/calendar?year=' + - str(eventYear) + - '?month=' + - str(eventMonthNumber) + - '?day=' + - str(eventDayOfMonth)) - calendarNotificationFile.close() return True diff --git a/inbox.py b/inbox.py index 9e9751f26..0c24e586c 100644 --- a/inbox.py +++ b/inbox.py @@ -83,6 +83,7 @@ from categories import guessHashtagCategory from context import hasValidContext from speaker import updateSpeaker from announce import isSelfAnnounce +from storage import storeValue def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -127,10 +128,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: daysSinceEpoch = daysDiff.days tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n' if not os.path.isfile(tagsFilename): - tagsFile = open(tagsFilename, "w+") - if tagsFile: - tagsFile.write(tagline) - tagsFile.close() + storeValue(tagsFilename, tagline, 'write') else: if postUrl not in open(tagsFilename).read(): try: @@ -1460,10 +1458,7 @@ def _receiveAnnounce(recentPostsCache: {}, postJsonObject, personCache, translate, lookupActor, themeName) - ttsFile = open(postFilename + '.tts', "w+") - if ttsFile: - ttsFile.write('\n') - ttsFile.close() + storeValue(postFilename + '.tts', '\n', 'writeonly') if debug: print('DEBUG: Obtaining actor for announce post ' + @@ -1642,15 +1637,9 @@ def populateReplies(baseDir: str, httpPrefix: str, domain: str, if numLines > maxReplies: return False if messageId not in open(postRepliesFilename).read(): - repliesFile = open(postRepliesFilename, 'a+') - if repliesFile: - repliesFile.write(messageId + '\n') - repliesFile.close() + storeValue(postRepliesFilename, messageId, 'append') else: - repliesFile = open(postRepliesFilename, 'w+') - if repliesFile: - repliesFile.write(messageId + '\n') - repliesFile.close() + storeValue(postRepliesFilename, messageId, 'write') return True @@ -1814,8 +1803,7 @@ def _dmNotify(baseDir: str, handle: str, url: str) -> None: return dmFile = accountDir + '/.newDM' if not os.path.isfile(dmFile): - with open(dmFile, 'w+') as fp: - fp.write(url) + storeValue(dmFile, url, 'writeonly') def _alreadyLiked(baseDir: str, nickname: str, domain: str, @@ -1895,20 +1883,8 @@ def _likeNotify(baseDir: str, domain: str, onionDomain: str, prevLikeStr = fp.read() if prevLikeStr == likeStr: return - try: - with open(prevLikeFile, 'w+') as fp: - fp.write(likeStr) - except BaseException: - print('ERROR: unable to save previous like notification ' + - prevLikeFile) - pass - try: - with open(likeFile, 'w+') as fp: - fp.write(likeStr) - except BaseException: - print('ERROR: unable to write like notification file ' + - likeFile) - pass + storeValue(prevLikeFile, likeStr, 'writeonly') + storeValue(likeFile, likeStr, 'writeonly') def _replyNotify(baseDir: str, handle: str, url: str) -> None: @@ -1919,8 +1895,7 @@ def _replyNotify(baseDir: str, handle: str, url: str) -> None: return replyFile = accountDir + '/.newReply' if not os.path.isfile(replyFile): - with open(replyFile, 'w+') as fp: - fp.write(url) + storeValue(replyFile, url, 'writeonly') def _gitPatchNotify(baseDir: str, handle: str, @@ -1934,8 +1909,7 @@ def _gitPatchNotify(baseDir: str, handle: str, patchFile = accountDir + '/.newPatch' subject = subject.replace('[PATCH]', '').strip() handle = '@' + fromNickname + '@' + fromDomain - with open(patchFile, 'w+') as fp: - fp.write('git ' + handle + ' ' + subject) + storeValue(patchFile, 'git ' + handle + ' ' + subject, 'writeonly') def _groupHandle(baseDir: str, handle: str) -> bool: @@ -2106,13 +2080,7 @@ def inboxUpdateIndex(boxname: str, baseDir: str, handle: str, except Exception as e: print('WARN: Failed to write entry to index ' + str(e)) else: - try: - indexFile = open(indexFilename, 'w+') - if indexFile: - indexFile.write(destinationFilename + '\n') - indexFile.close() - except Exception as e: - print('WARN: Failed to write initial entry to index ' + str(e)) + storeValue(indexFilename, destinationFilename, 'write') return False @@ -2145,8 +2113,8 @@ def _updateLastSeen(baseDir: str, handle: str, actor: str) -> None: if int(daysSinceEpochFile) == daysSinceEpoch: # value hasn't changed, so we can save writing anything to file return - with open(lastSeenFilename, 'w+') as lastSeenFile: - lastSeenFile.write(str(daysSinceEpoch)) + daysSinceEpochStr = str(daysSinceEpoch) + storeValue(lastSeenFilename, daysSinceEpochStr, 'writeonly') def _bounceDM(senderPostId: str, session, httpPrefix: str, @@ -2590,10 +2558,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, # This enables you to ignore a threat that's getting boring if isReplyToMutedPost: print('MUTE REPLY: ' + destinationFilename) - muteFile = open(destinationFilename + '.muted', 'w+') - if muteFile: - muteFile.write('\n') - muteFile.close() + storeValue(destinationFilename + '.muted', '\n', 'writeonly') # update the indexes for different timelines for boxname in updateIndexList: diff --git a/media.py b/media.py index 8b83d4f0d..5530f60f5 100644 --- a/media.py +++ b/media.py @@ -21,6 +21,7 @@ from shutil import copyfile from shutil import rmtree from shutil import move from city import spoofGeolocation +from storage import storeValue def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None: @@ -73,11 +74,8 @@ def _spoofMetaData(baseDir: str, nickname: str, domain: str, decoySeed = int(fp.read()) else: decoySeed = randint(10000, 10000000000000000) - try: - with open(decoySeedFilename, 'w+') as fp: - fp.write(str(decoySeed)) - except BaseException: - pass + decoySeedStr = str(decoySeed) + storeValue(decoySeedFilename, decoySeedStr, 'writeonly') if os.path.isfile('/usr/bin/exiftool'): print('Spoofing metadata in ' + outputFilename + ' using exiftool') @@ -192,11 +190,7 @@ def _updateEtag(mediaFilename: str) -> None: # calculate hash etag = sha1(data).hexdigest() # nosec # save the hash - try: - with open(mediaFilename + '.etag', 'w+') as etagFile: - etagFile.write(etag) - except BaseException: - pass + storeValue(mediaFilename + '.etag', etag, 'writeonly') def attachMedia(baseDir: str, httpPrefix: str, diff --git a/migrate.py b/migrate.py index 4cca4b0e8..9b03857fa 100644 --- a/migrate.py +++ b/migrate.py @@ -15,6 +15,7 @@ from blocking import isBlocked from session import getJson from posts import getUserUrl from follow import unfollowAccount +from storage import storeValue def _moveFollowingHandlesForAccount(baseDir: str, nickname: str, domain: str, @@ -148,11 +149,11 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, # save the new handles to the refollow list if os.path.isfile(refollowFilename): - with open(refollowFilename, 'a+') as f: - f.write(movedToHandle + '\n') + storeValue(refollowFilename, + movedToHandle, 'append') else: - with open(refollowFilename, 'w+') as f: - f.write(movedToHandle + '\n') + storeValue(refollowFilename, + movedToHandle, 'write') followersFilename = \ baseDir + '/accounts/' + nickname + '@' + domain + '/followers.txt' diff --git a/newsdaemon.py b/newsdaemon.py index bc8e4f5f0..c816417dd 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -34,6 +34,7 @@ from utils import clearFromPostCaches from utils import dangerousMarkup from inbox import storeHashTags from session import createSession +from storage import storeValue def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: @@ -55,19 +56,13 @@ def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: print('WARN: Failed to write entry to feeds posts index ' + indexFilename + ' ' + str(e)) else: - feedsFile = open(indexFilename, 'w+') - if feedsFile: - feedsFile.write(postId + '\n') - feedsFile.close() + storeValue(indexFilename, postId, 'write') def _saveArrivedTime(baseDir: str, postFilename: str, arrived: str) -> None: """Saves the time when an rss post arrived to a file """ - arrivedFile = open(postFilename + '.arrived', 'w+') - if arrivedFile: - arrivedFile.write(arrived) - arrivedFile.close() + storeValue(postFilename + '.arrived', arrived, 'writeonly') def _removeControlCharacters(content: str) -> str: @@ -409,8 +404,7 @@ def _createNewsMirror(baseDir: str, domain: str, for removePostId in removals: indexContent = \ indexContent.replace(removePostId + '\n', '') - with open(mirrorIndexFilename, "w+") as indexFile: - indexFile.write(indexContent) + storeValue(mirrorIndexFilename, indexContent, 'writeonly') mirrorArticleDir = mirrorDir + '/' + postIdNumber if os.path.isdir(mirrorArticleDir): @@ -435,15 +429,9 @@ def _createNewsMirror(baseDir: str, domain: str, # append the post Id number to the index file if os.path.isfile(mirrorIndexFilename): - indexFile = open(mirrorIndexFilename, "a+") - if indexFile: - indexFile.write(postIdNumber + '\n') - indexFile.close() + storeValue(mirrorIndexFilename, postIdNumber, 'append') else: - indexFile = open(mirrorIndexFilename, "w+") - if indexFile: - indexFile.write(postIdNumber + '\n') - indexFile.close() + storeValue(mirrorIndexFilename, postIdNumber, 'write') return True diff --git a/person.py b/person.py index 0771bd115..9cb0af29b 100644 --- a/person.py +++ b/person.py @@ -52,6 +52,7 @@ from session import createSession from session import getJson from webfinger import webfingerHandle from pprint import pprint +from storage import storeValue def generateRSAKey() -> (str, str): @@ -494,15 +495,13 @@ def createPerson(baseDir: str, nickname: str, domain: str, port: int, if manualFollowerApproval: followDMsFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.followDMs' - with open(followDMsFilename, 'w+') as fFile: - fFile.write('\n') + storeValue(followDMsFilename, '\n', 'writeonly') # notify when posts are liked if nickname != 'news': notifyLikesFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.notifyLikes' - with open(notifyLikesFilename, 'w+') as nFile: - nFile.write('\n') + storeValue(notifyLikesFilename, '\n', 'writeonly') theme = getConfigParam(baseDir, 'theme') if not theme: @@ -923,15 +922,9 @@ def suspendAccount(baseDir: str, nickname: str, domain: str) -> None: for suspended in lines: if suspended.strip('\n').strip('\r') == nickname: return - suspendedFile = open(suspendedFilename, 'a+') - if suspendedFile: - suspendedFile.write(nickname + '\n') - suspendedFile.close() + storeValue(suspendedFilename, nickname, 'append') else: - suspendedFile = open(suspendedFilename, 'w+') - if suspendedFile: - suspendedFile.write(nickname + '\n') - suspendedFile.close() + storeValue(suspendedFilename, nickname, 'write') def canRemovePost(baseDir: str, nickname: str, @@ -1132,10 +1125,7 @@ def isPersonSnoozed(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - writeSnoozedFile = open(snoozedFilename, 'w+') - if writeSnoozedFile: - writeSnoozedFile.write(content) - writeSnoozedFile.close() + storeValue(snoozedFilename, content, 'writeonly') if snoozeActor + ' ' in open(snoozedFilename).read(): return True @@ -1185,10 +1175,7 @@ def personUnsnooze(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - writeSnoozedFile = open(snoozedFilename, 'w+') - if writeSnoozedFile: - writeSnoozedFile.write(content) - writeSnoozedFile.close() + storeValue(snoozedFilename, content, 'writeonly') def setPersonNotes(baseDir: str, nickname: str, domain: str, @@ -1204,8 +1191,7 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, if not os.path.isdir(notesDir): os.mkdir(notesDir) notesFilename = notesDir + '/' + handle + '.txt' - with open(notesFilename, 'w+') as notesFile: - notesFile.write(notes) + storeValue(notesFilename, notes, 'writeonly') return True diff --git a/petnames.py b/petnames.py index 21c46095a..52ece3c3f 100644 --- a/petnames.py +++ b/petnames.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from storage import storeValue def setPetName(baseDir: str, nickname: str, domain: str, @@ -40,17 +41,14 @@ def setPetName(baseDir: str, nickname: str, domain: str, else: newPetnamesStr += entry # save the updated petnames file - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(newPetnamesStr) + storeValue(petnamesFilename, newPetnamesStr, 'writeonly') return True # entry does not exist in the petnames file - with open(petnamesFilename, 'a+') as petnamesFile: - petnamesFile.write(entry) + storeValue(petnamesFilename, entry, 'append') return True # first entry - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(entry) + storeValue(petnamesFilename, entry, 'writeonly') return True diff --git a/posts.py b/posts.py index 6e275d45f..05291fedf 100644 --- a/posts.py +++ b/posts.py @@ -71,6 +71,7 @@ from filters import isFiltered from git import convertPostToPatch from linked_data_sig import generateJsonSignature from petnames import resolvePetnames +from storage import storeValue def isModerator(baseDir: str, nickname: str) -> bool: @@ -733,17 +734,7 @@ def _updateHashtagsIndex(baseDir: str, tag: {}, newPostId: str) -> None: tagsFile.write(tagline) tagsFile.close() else: - # prepend to tags index file - if tagline not in open(tagsFilename).read(): - try: - with open(tagsFilename, 'r+') as tagsFile: - content = tagsFile.read() - if tagline not in content: - tagsFile.seek(0, 0) - tagsFile.write(tagline + content) - except Exception as e: - print('WARN: Failed to write entry to tags file ' + - tagsFilename + ' ' + str(e)) + storeValue(tagsFilename, tagline, 'prepend') def _addSchedulePost(baseDir: str, nickname: str, domain: str, @@ -767,10 +758,7 @@ def _addSchedulePost(baseDir: str, nickname: str, domain: str, print('WARN: Failed to write entry to scheduled posts index ' + scheduleIndexFilename + ' ' + str(e)) else: - scheduleFile = open(scheduleIndexFilename, 'w+') - if scheduleFile: - scheduleFile.write(indexStr + '\n') - scheduleFile.close() + storeValue(scheduleIndexFilename, indexStr, 'write') def _appendEventFields(newPost: {}, @@ -1194,10 +1182,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, newPost['moderationStatus'] = 'pending' # save to index file moderationIndexFile = baseDir + '/accounts/moderation.txt' - modFile = open(moderationIndexFile, "a+") - if modFile: - modFile.write(newPostId + '\n') - modFile.close() + storeValue(moderationIndexFile, newPostId, 'append') # If a patch has been posted - i.e. the output from # git format-patch - then convert the activitypub type @@ -1305,10 +1290,7 @@ def pinPost(baseDir: str, nickname: str, domain: str, """ accountDir = baseDir + '/accounts/' + nickname + '@' + domain pinnedFilename = accountDir + '/pinToProfile.txt' - pinFile = open(pinnedFilename, "w+") - if pinFile: - pinFile.write(pinnedContent) - pinFile.close() + storeValue(pinnedFilename, pinnedContent, 'writeonly') def undoPinnedPost(baseDir: str, nickname: str, domain: str) -> None: @@ -1850,11 +1832,7 @@ def createReportPost(baseDir: str, newReportFile = baseDir + '/accounts/' + handle + '/.newReport' if os.path.isfile(newReportFile): continue - try: - with open(newReportFile, 'w+') as fp: - fp.write(toUrl + '/moderation') - except BaseException: - pass + storeValue(newReportFile, toUrl + '/moderation', 'writeonly') return postJsonObject @@ -1898,8 +1876,7 @@ def threadSendPost(session, postJsonStr: str, federationList: [], if debug: # save the log file postLogFilename = baseDir + '/post.log' - with open(postLogFilename, "a+") as logFile: - logFile.write(logStr + '\n') + storeValue(postLogFilename, logStr, 'append') if postResult: if debug: @@ -3452,10 +3429,7 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str, break # save the new index file if len(newIndex) > 0: - indexFile = open(indexFilename, 'w+') - if indexFile: - indexFile.write(newIndex) - indexFile.close() + storeValue(indexFilename, newIndex, 'writeonly') postsInBoxDict = {} postsCtr = 0 @@ -3838,8 +3812,7 @@ def checkDomains(session, baseDir: str, updateFollowerWarnings = True if updateFollowerWarnings and followerWarningStr: - with open(followerWarningFilename, 'w+') as fp: - fp.write(followerWarningStr) + storeValue(followerWarningFilename, followerWarningStr, 'writeonly') if not singleCheck: print(followerWarningStr) @@ -3919,10 +3892,7 @@ def _rejectAnnounce(announceFilename: str, # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): - rejectAnnounceFile = open(announceFilename + '.reject', "w+") - if rejectAnnounceFile: - rejectAnnounceFile.write('\n') - rejectAnnounceFile.close() + storeValue(announceFilename + '.reject', '\n', 'writeonly') def downloadAnnounce(session, baseDir: str, httpPrefix: str, diff --git a/question.py b/question.py index 7b7328f44..5d0e08fc5 100644 --- a/question.py +++ b/question.py @@ -11,6 +11,7 @@ import os from utils import locatePost from utils import loadJson from utils import saveJson +from storage import storeValue def questionUpdateVotes(baseDir: str, nickname: str, domain: str, @@ -67,21 +68,17 @@ def questionUpdateVotes(baseDir: str, nickname: str, domain: str, votersFilename = questionPostFilename.replace('.json', '.voters') if not os.path.isfile(votersFilename): # create a new voters file - votersFile = open(votersFilename, 'w+') - if votersFile: - votersFile.write(replyJson['actor'] + - votersFileSeparator + - foundAnswer + '\n') - votersFile.close() + vStr = replyJson['actor'] + \ + votersFileSeparator + \ + foundAnswer + storeValue(votersFilename, vStr, 'write') else: if replyJson['actor'] not in open(votersFilename).read(): # append to the voters file - votersFile = open(votersFilename, "a+") - if votersFile: - votersFile.write(replyJson['actor'] + - votersFileSeparator + - foundAnswer + '\n') - votersFile.close() + vStr = replyJson['actor'] + \ + votersFileSeparator + \ + foundAnswer + storeValue(votersFilename, vStr, 'append') else: # change an entry in the voters file with open(votersFilename, "r") as votersFile: diff --git a/shares.py b/shares.py index 4fb62daad..f3f1d2adf 100644 --- a/shares.py +++ b/shares.py @@ -20,6 +20,7 @@ from utils import loadJson from utils import saveJson from utils import getImageExtensions from media import processMetaData +from storage import storeValue def getValidSharedItemID(displayName: str) -> str: @@ -161,12 +162,10 @@ def addShare(baseDir: str, newShareFile = accountDir + '/.newShare' if not os.path.isfile(newShareFile): nickname = handle.split('@')[0] - try: - with open(newShareFile, 'w+') as fp: - fp.write(httpPrefix + '://' + domainFull + - '/users/' + nickname + '/tlshares') - except BaseException: - pass + storeValue(newShareFile, + httpPrefix + '://' + domainFull + + '/users/' + nickname + '/tlshares', + 'writeonly') break diff --git a/storage.py b/storage.py new file mode 100644 index 000000000..0417796f0 --- /dev/null +++ b/storage.py @@ -0,0 +1,56 @@ +__filename__ = "storage.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.2.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" +__module_group__ = "storage" + +import os + + +def storeValue(filename: str, lineStr: str, storeType: str) -> bool: + """Stores a line to a file + """ + if not lineStr.endswith('\n'): + if storeType != 'writeonly': + lineStr += '\n' + + if storeType[0] == 'a': + if not os.path.isfile(filename): + storeType = 'write' + + if storeType[0] == 'a': + if not os.path.isfile(filename): + return False + # append + try: + with open(filename, "a+") as fp: + fp.write(lineStr) + return True + except Exception as e: + print('ERROR: unable to append to ' + filename + ' ' + str(e)) + pass + elif storeType[0] == 'w': + # new file + try: + with open(filename, "w+") as fp: + fp.write(lineStr) + return True + except Exception as e: + print('ERROR: unable to write to ' + filename + ' ' + str(e)) + pass + elif storeType[0] == 'p': + # prepend + if lineStr not in open(filename).read(): + try: + with open(filename, 'r+') as fp: + content = fp.read() + if lineStr not in content: + fp.seek(0, 0) + fp.write(lineStr + content) + except Exception as e: + print('WARN: Unable to prepend to ' + + filename + ' ' + str(e)) + return False diff --git a/tests.py b/tests.py index f270ae9e1..8b18368f4 100644 --- a/tests.py +++ b/tests.py @@ -117,6 +117,7 @@ from mastoapiv1 import getNicknameFromMastoApiV1Id from webapp_post import prepareHtmlPostNickname from webapp_utils import markdownToHtml from speaker import speakerReplaceLinks +from storage import storeValue testServerAliceRunning = False testServerBobRunning = False @@ -3167,12 +3168,11 @@ def _testFunctions(): callGraphStr += ' }\n' clusterCtr += 1 callGraphStr += '\n}\n' - with open('epicyon_modules.dot', 'w+') as fp: - fp.write(callGraphStr) - print('Modules call graph saved to epicyon_modules.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + - '-Gsep=+100 -Tx11 epicyon_modules.dot') + assert storeValue('epicyon_modules.dot', callGraphStr, 'writeonly') + print('Modules call graph saved to epicyon_modules.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + + '-Gsep=+100 -Tx11 epicyon_modules.dot') callGraphStr = 'digraph Epicyon {\n\n' callGraphStr += ' size="8,6"; ratio=fill;\n' @@ -3223,12 +3223,11 @@ def _testFunctions(): '" [color=' + modColor + '];\n' callGraphStr += '\n}\n' - with open('epicyon.dot', 'w+') as fp: - fp.write(callGraphStr) - print('Call graph saved to epicyon.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + - '-Gsep=+120 -Tx11 epicyon.dot') + assert storeValue('epicyon.dot', callGraphStr, 'writeonly') + print('Call graph saved to epicyon.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + + '-Gsep=+120 -Tx11 epicyon.dot') def _testLinksWithinPost() -> None: @@ -3883,10 +3882,7 @@ def _testSpoofGeolocation() -> None: kmlStr += '\n' kmlStr += '' - kmlFile = open('unittest_decoy.kml', 'w+') - if kmlFile: - kmlFile.write(kmlStr) - kmlFile.close() + assert storeValue('unittest_decoy.kml', kmlStr, 'writeonly') def _testSkills() -> None: diff --git a/theme.py b/theme.py index 7fdb0d720..5fa2bac01 100644 --- a/theme.py +++ b/theme.py @@ -16,6 +16,7 @@ from shutil import make_archive from shutil import unpack_archive from shutil import rmtree from content import dangerousCSS +from storage import storeValue def importTheme(baseDir: str, filename: str) -> bool: @@ -361,8 +362,7 @@ def _setThemeFromDict(baseDir: str, name: str, continue css = setCSSparam(css, paramName, paramValue) filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') if bgParams.get('login'): _setBackgroundFormat(baseDir, name, 'login', bgParams['login']) @@ -388,8 +388,7 @@ def _setBackgroundFormat(baseDir: str, name: str, with open(cssFilename, 'r') as cssfile: css = cssfile.read() css = css.replace('background.jpg', 'background.' + extension) - with open(cssFilename, 'w+') as cssfile2: - cssfile2.write(css) + storeValue(cssFilename, css, 'writeonly') def enableGrayscale(baseDir: str) -> None: @@ -407,12 +406,10 @@ def enableGrayscale(baseDir: str) -> None: css.replace('body, html {', 'body, html {\n filter: grayscale(100%);') filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') grayscaleFilename = baseDir + '/accounts/.grayscale' if not os.path.isfile(grayscaleFilename): - with open(grayscaleFilename, 'w+') as grayfile: - grayfile.write(' ') + storeValue(grayscaleFilename, ' ', 'writeonly') def disableGrayscale(baseDir: str) -> None: @@ -429,8 +426,7 @@ def disableGrayscale(baseDir: str) -> None: css = \ css.replace('\n filter: grayscale(100%);', '') filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') grayscaleFilename = baseDir + '/accounts/.grayscale' if os.path.isfile(grayscaleFilename): os.remove(grayscaleFilename) @@ -470,8 +466,7 @@ def _setCustomFont(baseDir: str): customFontType + "')") css = setCSSparam(css, "*font-family", "'CustomFont'") filename = baseDir + '/' + filename - with open(filename, 'w+') as cssfile: - cssfile.write(css) + storeValue(filename, css, 'writeonly') def _readVariablesFile(baseDir: str, themeName: str, @@ -739,8 +734,7 @@ def _setClearCacheFlag(baseDir: str) -> None: if not os.path.isdir(baseDir + '/accounts'): return flagFilename = baseDir + '/accounts/.clear_cache' - with open(flagFilename, 'w+') as flagFile: - flagFile.write('\n') + storeValue(flagFilename, '\n', 'writeonly') def setTheme(baseDir: str, name: str, domain: str, diff --git a/utils.py b/utils.py index fd159e8af..e22a2f68d 100644 --- a/utils.py +++ b/utils.py @@ -18,6 +18,7 @@ from pprint import pprint from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes +from storage import storeValue # posts containing these strings will always get screened out, # both incoming and outgoing. @@ -43,9 +44,7 @@ def refreshNewswire(baseDir: str): refreshNewswireFilename = baseDir + '/accounts/.refresh_newswire' if os.path.isfile(refreshNewswireFilename): return - refreshFile = open(refreshNewswireFilename, 'w+') - refreshFile.write('\n') - refreshFile.close() + storeValue(refreshNewswireFilename, '\n', 'writeonly') def getSHA256(msg: str): @@ -490,15 +489,13 @@ def saveJson(jsonObject: {}, filename: str) -> bool: """Saves json to a file """ tries = 0 + storeStr = json.dumps(jsonObject) while tries < 5: - try: - with open(filename, 'w+') as fp: - fp.write(json.dumps(jsonObject)) - return True - except BaseException: - print('WARN: saveJson ' + str(tries)) - time.sleep(1) - tries += 1 + if storeValue(filename, storeStr, 'writeonly'): + return True + print('WARN: saveJson ' + str(tries)) + time.sleep(1) + tries += 1 return False @@ -942,8 +939,7 @@ def _setDefaultPetName(baseDir: str, nickname: str, domain: str, followNickname + '@' + followDomain + '\n' if not os.path.isfile(petnamesFilename): # if there is no existing petnames lookup file - with open(petnamesFilename, 'w+') as petnamesFile: - petnamesFile.write(petnameLookupEntry) + storeValue(petnamesFilename, petnameLookupEntry, 'writeonly') return with open(petnamesFilename, 'r') as petnamesFile: @@ -1000,8 +996,7 @@ def followPerson(baseDir: str, nickname: str, domain: str, for line in lines: if handleToFollow not in line: newLines += line - with open(unfollowedFilename, 'w+') as f: - f.write(newLines) + storeValue(unfollowedFilename, newLines, 'writeonly') if not os.path.isdir(baseDir + '/accounts'): os.mkdir(baseDir + '/accounts') @@ -1029,8 +1024,7 @@ def followPerson(baseDir: str, nickname: str, domain: str, print('DEBUG: ' + handle + ' creating new following file to follow ' + handleToFollow + ', filename is ' + filename) - with open(filename, 'w+') as f: - f.write(handleToFollow + '\n') + storeValue(filename, handleToFollow, 'write') if followFile.endswith('following.txt'): # Default to adding new follows to the calendar. @@ -1352,8 +1346,7 @@ def deletePost(baseDir: str, httpPrefix: str, # hashtag file os.remove(tagIndexFilename) else: - with open(tagIndexFilename, "w+") as f: - f.write(newlines) + storeValue(tagIndexFilename, newlines, 'writeonly') # remove any replies repliesFilename = postFilename.replace('.json', '.replies') @@ -2198,10 +2191,7 @@ def rejectPostId(baseDir: str, nickname: str, domain: str, if recentPostsCache['html'].get(postUrl): del recentPostsCache['html'][postUrl] - rejectFile = open(postFilename + '.reject', "w+") - if rejectFile: - rejectFile.write('\n') - rejectFile.close() + storeValue(postFilename + '.reject', '\n', 'writeonly') def isDM(postJsonObject: {}) -> bool: diff --git a/webapp_post.py b/webapp_post.py index f1f845b73..e96689be2 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -69,6 +69,7 @@ from webapp_question import insertQuestion from devices import E2EEdecryptMessageFromDevice from webfinger import webfingerHandle from speaker import updateSpeaker +from storage import storeValue def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None: @@ -156,13 +157,7 @@ def _saveIndividualPostAsHtmlToCache(baseDir: str, if not os.path.isdir(htmlPostCacheDir): os.mkdir(htmlPostCacheDir) - try: - with open(cachedPostFilename, 'w+') as fp: - fp.write(postHtml) - return True - except Exception as e: - print('ERROR: saving post to cache ' + str(e)) - return False + return storeValue(cachedPostFilename, postHtml, 'writeonly') def _getPostFromRecentCache(session, @@ -1332,10 +1327,8 @@ def individualPostAsHtml(allowDownloads: bool, postJsonObject, personCache, translate, postJsonObject['actor'], themeName) - ttsFile = open(announceFilename + '.tts', "w+") - if ttsFile: - ttsFile.write('\n') - ttsFile.close() + storeValue(announceFilename + '.tts', + '\n', 'writeonly') isAnnounced = True diff --git a/webapp_utils.py b/webapp_utils.py index 7797d040c..1c611473d 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -20,6 +20,7 @@ from cache import getPersonFromCache from cache import storePersonInCache from content import addHtmlTags from content import replaceEmojiFromTags +from storage import storeValue def _markdownEmphasisHtml(markdown: str) -> str: @@ -1387,5 +1388,4 @@ def setMinimal(baseDir: str, domain: str, nickname: str, if minimal and minimalFileExists: os.remove(minimalFilename) elif not minimal and not minimalFileExists: - with open(minimalFilename, 'w+') as fp: - fp.write('\n') + storeValue(minimalFilename, '\n', 'writeonly') diff --git a/webapp_welcome.py b/webapp_welcome.py index 78e1cf441..9bd0f5111 100644 --- a/webapp_welcome.py +++ b/webapp_welcome.py @@ -14,6 +14,7 @@ from utils import removeHtml from webapp_utils import htmlHeaderWithExternalStyle from webapp_utils import htmlFooter from webapp_utils import markdownToHtml +from storage import storeValue def isWelcomeScreenComplete(baseDir: str, nickname: str, domain: str) -> bool: @@ -34,10 +35,7 @@ def welcomeScreenIsComplete(baseDir: str, if not os.path.isdir(accountPath): return completeFilename = accountPath + '/.welcome_complete' - completeFile = open(completeFilename, 'w+') - if completeFile: - completeFile.write('\n') - completeFile.close() + storeValue(completeFilename, '\n', 'writeonly') def htmlWelcomeScreen(baseDir: str, nickname: str,