From d42ee647d3265dd4d436e31fcd9c3316bcf6d4af Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 21 Jun 2021 23:53:04 +0100 Subject: [PATCH] Revert "File storage functions" This reverts commit 9c18e7042e76c6a1282b4f0ab86c58eeeaf197d4. --- auth.py | 26 ++++--- blocking.py | 6 +- bookmarks.py | 11 ++- categories.py | 7 +- daemon.py | 158 ++++++++++++++++++++++++++++--------------- desktop_client.py | 6 +- epicyon.py | 9 ++- follow.py | 14 ++-- followingCalendar.py | 13 ++-- git.py | 7 +- happening.py | 31 +++++---- inbox.py | 63 +++++++++++++---- media.py | 14 ++-- migrate.py | 9 ++- newsdaemon.py | 24 +++++-- person.py | 30 +++++--- petnames.py | 10 +-- posts.py | 50 +++++++++++--- question.py | 21 +++--- shares.py | 11 +-- storage.py | 56 --------------- tests.py | 28 ++++---- theme.py | 22 +++--- utils.py | 36 ++++++---- webapp_post.py | 15 ++-- webapp_utils.py | 4 +- webapp_welcome.py | 6 +- 27 files changed, 420 insertions(+), 267 deletions(-) delete mode 100644 storage.py diff --git a/auth.py b/auth.py index c62dc8236..87b94e5e2 100644 --- a/auth.py +++ b/auth.py @@ -15,7 +15,6 @@ import secrets import datetime from utils import isSystemAccount from utils import hasUsersPath -from storage import storeValue def _hashPassword(password: str) -> str: @@ -176,7 +175,8 @@ def storeBasicCredentials(baseDir: str, nickname: str, password: str) -> bool: with open(passwordFile, 'a+') as passfile: passfile.write(storeStr + '\n') else: - storeValue(passwordFile, storeStr, 'write') + with open(passwordFile, 'w+') as passfile: + passfile.write(storeStr + '\n') return True @@ -240,14 +240,18 @@ def recordLoginFailure(baseDir: str, ipAddress: str, return failureLog = baseDir + '/accounts/loginfailures.log' - writeType = 'append' + writeType = 'a+' if not os.path.isfile(failureLog): - writeType = 'writeonly' + writeType = 'w+' currTime = datetime.datetime.utcnow() - logLineStr = \ - currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + \ - 'ip-127-0-0-1 sshd[20710]: ' + \ - 'Disconnecting invalid user epicyon ' + \ - ipAddress + ' port 443: ' + \ - 'Too many authentication failures [preauth]\n' - storeValue(failureLog, logLineStr, writeType) + try: + with open(failureLog, writeType) as fp: + # here we use a similar format to an ssh log, so that + # systems such as fail2ban can parse it + fp.write(currTime.strftime("%Y-%m-%d %H:%M:%SZ") + ' ' + + 'ip-127-0-0-1 sshd[20710]: ' + + 'Disconnecting invalid user epicyon ' + + ipAddress + ' port 443: ' + + 'Too many authentication failures [preauth]\n') + except BaseException: + pass diff --git a/blocking.py b/blocking.py index a90a63953..86a6c052d 100644 --- a/blocking.py +++ b/blocking.py @@ -25,7 +25,6 @@ from utils import locatePost from utils import evilIncarnate from utils import getDomainFromActor from utils import getNicknameFromActor -from storage import storeValue def addGlobalBlock(baseDir: str, @@ -494,7 +493,10 @@ def mutePost(baseDir: str, nickname: str, domain: str, port: int, if os.path.isfile(cachedPostFilename): os.remove(cachedPostFilename) - if storeValue(postFilename + '.muted', '\n', 'writeonly'): + muteFile = open(postFilename + '.muted', 'w+') + if muteFile: + muteFile.write('\n') + muteFile.close() print('MUTE: ' + postFilename + '.muted file added') # if the post is in the recent posts cache then mark it as muted diff --git a/bookmarks.py b/bookmarks.py index 3858d22e7..58a772b3d 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -24,7 +24,6 @@ from utils import loadJson from utils import saveJson from posts import getPersonBox from session import postJson -from storage import storeValue def undoBookmarksCollectionEntry(recentPostsCache: {}, @@ -62,7 +61,10 @@ def undoBookmarksCollectionEntry(recentPostsCache: {}, indexStr = '' with open(bookmarksIndexFilename, 'r') as indexFile: indexStr = indexFile.read().replace(bookmarkIndex + '\n', '') - storeValue(bookmarksIndexFilename, indexStr, 'writeonly') + bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') + if bookmarksIndexFile: + bookmarksIndexFile.write(indexStr) + bookmarksIndexFile.close() if not postJsonObject.get('type'): return @@ -217,7 +219,10 @@ def updateBookmarksCollection(recentPostsCache: {}, print('WARN: Failed to write entry to bookmarks index ' + bookmarksIndexFilename + ' ' + str(e)) else: - storeValue(bookmarksIndexFilename, bookmarkIndex, 'write') + bookmarksIndexFile = open(bookmarksIndexFilename, 'w+') + if bookmarksIndexFile: + bookmarksIndexFile.write(bookmarkIndex + '\n') + bookmarksIndexFile.close() def bookmark(recentPostsCache: {}, diff --git a/categories.py b/categories.py index 67c65ba15..70d079a78 100644 --- a/categories.py +++ b/categories.py @@ -9,7 +9,6 @@ __module_group__ = "RSS Feeds" import os import datetime -from storage import storeValue def getHashtagCategory(baseDir: str, hashtag: str) -> str: @@ -107,7 +106,8 @@ def _updateHashtagCategories(baseDir: str) -> None: categoryListStr += categoryStr + '\n' # save a list of available categories for quick lookup - storeValue(categoryListFilename, categoryListStr, 'writeonly') + with open(categoryListFilename, 'w+') as fp: + fp.write(categoryListStr) def _validHashtagCategory(category: str) -> bool: @@ -153,7 +153,8 @@ def setHashtagCategory(baseDir: str, hashtag: str, category: str, # don't overwrite any existing categories if os.path.isfile(categoryFilename): return False - if storeValue(categoryFilename, category, 'writeonly'): + with open(categoryFilename, 'w+') as fp: + fp.write(category) _updateHashtagCategories(baseDir) return True diff --git a/daemon.py b/daemon.py index db2cffa52..223eace39 100644 --- a/daemon.py +++ b/daemon.py @@ -300,7 +300,6 @@ from context import hasValidContext from speaker import getSSMLbox from city import getSpoofedCity import os -from storage import storeValue # maximum number of posts to list in outbox feed @@ -675,7 +674,11 @@ class PubServer(BaseHTTPRequestHandler): pass if not etag: etag = sha1(data).hexdigest() # nosec - storeValue(mediaFilename + '.etag', etag, 'writeonly') + try: + with open(mediaFilename + '.etag', 'w+') as etagFile: + etagFile.write(etag) + except BaseException: + pass if etag: self.send_header('ETag', etag) self.end_headers() @@ -1542,7 +1545,12 @@ class PubServer(BaseHTTPRequestHandler): print('WARN: Unable to read salt for ' + loginNickname + ' ' + str(e)) else: - storeValue(saltFilename, salt, 'writeonly') + try: + with open(saltFilename, 'w+') as fp: + fp.write(salt) + except Exception as e: + print('WARN: Unable to save salt for ' + + loginNickname + ' ' + str(e)) tokenText = loginNickname + loginPassword + salt token = sha256(tokenText.encode('utf-8')).hexdigest() @@ -1551,7 +1559,12 @@ class PubServer(BaseHTTPRequestHandler): tokenFilename = \ baseDir+'/accounts/' + \ loginHandle + '/.token' - storeValue(tokenFilename, token, 'writeonly') + try: + with open(tokenFilename, 'w+') as fp: + fp.write(token) + except Exception as e: + print('WARN: Unable to save token for ' + + loginNickname + ' ' + str(e)) personUpgradeActor(baseDir, None, loginHandle, baseDir + '/accounts/' + @@ -2091,8 +2104,10 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - if storeValue(newswireBlockedFilename, - '\n', 'writeonly'): + noNewswireFile = open(newswireBlockedFilename, "w+") + if noNewswireFile: + noNewswireFile.write('\n') + noNewswireFile.close() refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2125,8 +2140,10 @@ class PubServer(BaseHTTPRequestHandler): refreshNewswire(self.server.baseDir) else: if os.path.isdir(accountDir): - if storeValue(featuresBlockedFilename, - '\n', 'writeonly'): + noFeaturesFile = open(featuresBlockedFilename, "w+") + if noFeaturesFile: + noFeaturesFile.write('\n') + noFeaturesFile.close() refreshNewswire(self.server.baseDir) usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ @@ -2158,7 +2175,10 @@ class PubServer(BaseHTTPRequestHandler): os.remove(newswireModFilename) else: if os.path.isdir(accountDir): - storeValue(newswireModFilename, '\n', 'writeonly') + modNewswireFile = open(newswireModFilename, "w+") + if modNewswireFile: + modNewswireFile.write('\n') + modNewswireFile.close() usersPathStr = \ usersPath + '/' + self.server.defaultTimeline + \ '?page=' + str(pageNumber) @@ -3439,7 +3459,10 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('editedLinks'): linksStr = fields['editedLinks'] - storeValue(linksFilename, linksStr, 'writeonly') + linksFile = open(linksFilename, "w+") + if linksFile: + linksFile.write(linksStr) + linksFile.close() else: if os.path.isfile(linksFilename): os.remove(linksFilename) @@ -3451,7 +3474,10 @@ class PubServer(BaseHTTPRequestHandler): aboutStr = fields['editedAbout'] if not dangerousMarkup(aboutStr, allowLocalNetworkAccess): - storeValue(aboutFilename, aboutStr, 'writeonly') + aboutFile = open(aboutFilename, "w+") + if aboutFile: + aboutFile.write(aboutStr) + aboutFile.close() else: if os.path.isfile(aboutFilename): os.remove(aboutFilename) @@ -3460,7 +3486,10 @@ class PubServer(BaseHTTPRequestHandler): TOSStr = fields['editedTOS'] if not dangerousMarkup(TOSStr, allowLocalNetworkAccess): - storeValue(TOSFilename, TOSStr, 'writeonly') + TOSFile = open(TOSFilename, "w+") + if TOSFile: + TOSFile.write(TOSStr) + TOSFile.close() else: if os.path.isfile(TOSFilename): os.remove(TOSFilename) @@ -3635,7 +3664,10 @@ class PubServer(BaseHTTPRequestHandler): extractTextFieldsInPOST(postBytes, boundary, debug) if fields.get('editedNewswire'): newswireStr = fields['editedNewswire'] - storeValue(newswireFilename, newswireStr, 'writeonly') + newswireFile = open(newswireFilename, "w+") + if newswireFile: + newswireFile.write(newswireStr) + newswireFile.close() else: if os.path.isfile(newswireFilename): os.remove(newswireFilename) @@ -3645,8 +3677,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ 'news@' + domain + '/filters.txt' if fields.get('filteredWordsNewswire'): - storeValue(filterNewswireFilename, - fields['filteredWordsNewswire'], 'writeonly') + with open(filterNewswireFilename, 'w+') as filterfile: + filterfile.write(fields['filteredWordsNewswire']) else: if os.path.isfile(filterNewswireFilename): os.remove(filterNewswireFilename) @@ -3655,8 +3687,8 @@ class PubServer(BaseHTTPRequestHandler): hashtagRulesFilename = \ baseDir + '/accounts/hashtagrules.txt' if fields.get('hashtagRulesList'): - storeValue(hashtagRulesFilename, - fields['hashtagRulesList'], 'writeonly') + with open(hashtagRulesFilename, 'w+') as rulesfile: + rulesfile.write(fields['hashtagRulesList']) else: if os.path.isfile(hashtagRulesFilename): os.remove(hashtagRulesFilename) @@ -3666,8 +3698,10 @@ class PubServer(BaseHTTPRequestHandler): newswireTrusted = fields['trustedNewswire'] if not newswireTrusted.endswith('\n'): newswireTrusted += '\n' - storeValue(newswireTrustedFilename, - newswireTrusted, 'writeonly') + trustFile = open(newswireTrustedFilename, "w+") + if trustFile: + trustFile.write(newswireTrusted) + trustFile.close() else: if os.path.isfile(newswireTrustedFilename): os.remove(newswireTrustedFilename) @@ -3753,8 +3787,10 @@ class PubServer(BaseHTTPRequestHandler): citationsStr += citationDate + '\n' # save citations dates, so that they can be added when # reloading the newblog screen - storeValue(citationsFilename, - citationsStr, 'writeonly') + citationsFile = open(citationsFilename, "w+") + if citationsFile: + citationsFile.write(citationsStr) + citationsFile.close() # redirect back to the default timeline self._redirect_headers(actorStr + '/newblog', @@ -4190,8 +4226,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('cityDropdown'): cityFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/city.txt' - storeValue(cityFilename, - fields['cityDropdown'], 'writeonly') + with open(cityFilename, 'w+') as fp: + fp.write(fields['cityDropdown']) # change displayed name if fields.get('displayNickname'): @@ -4964,15 +5000,16 @@ class PubServer(BaseHTTPRequestHandler): if onFinalWelcomeScreen: # initial default setting created via # the welcome screen - storeValue(followDMsFilename, '\n', 'writeonly') + with open(followDMsFilename, 'w+') as fFile: + fFile.write('\n') actorChanged = True else: followDMsActive = False if fields.get('followDMs'): if fields['followDMs'] == 'on': followDMsActive = True - storeValue(followDMsFilename, - '\n', 'writeonly') + with open(followDMsFilename, 'w+') as fFile: + fFile.write('\n') if not followDMsActive: if os.path.isfile(followDMsFilename): os.remove(followDMsFilename) @@ -4986,8 +5023,9 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('removeTwitter'): if fields['removeTwitter'] == 'on': removeTwitterActive = True - storeValue(removeTwitterFilename, - '\n', 'writeonly') + with open(removeTwitterFilename, + 'w+') as rFile: + rFile.write('\n') if not removeTwitterActive: if os.path.isfile(removeTwitterFilename): os.remove(removeTwitterFilename) @@ -5005,8 +5043,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('hideLikeButton'): if fields['hideLikeButton'] == 'on': hideLikeButtonActive = True - storeValue(hideLikeButtonFile, - '\n', 'writeonly') + with open(hideLikeButtonFile, 'w+') as rFile: + rFile.write('\n') # remove notify likes selection if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5017,8 +5055,8 @@ class PubServer(BaseHTTPRequestHandler): # notify about new Likes if onFinalWelcomeScreen: # default setting from welcome screen - storeValue(notifyLikesFilename, - '\n', 'writeonly') + with open(notifyLikesFilename, 'w+') as rFile: + rFile.write('\n') actorChanged = True else: notifyLikesActive = False @@ -5026,8 +5064,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['notifyLikes'] == 'on' and \ not hideLikeButtonActive: notifyLikesActive = True - storeValue(notifyLikesFilename, - '\n', 'writeonly') + with open(notifyLikesFilename, 'w+') as rFile: + rFile.write('\n') if not notifyLikesActive: if os.path.isfile(notifyLikesFilename): os.remove(notifyLikesFilename) @@ -5070,8 +5108,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/filters.txt' if fields.get('filteredWords'): - storeValue(filterFilename, - fields['filteredWords'], 'writeonly') + with open(filterFilename, 'w+') as filterfile: + filterfile.write(fields['filteredWords']) else: if os.path.isfile(filterFilename): os.remove(filterFilename) @@ -5082,8 +5120,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/replacewords.txt' if fields.get('switchWords'): - storeValue(switchFilename, - fields['switchWords'], 'writeonly') + with open(switchFilename, 'w+') as switchfile: + switchfile.write(fields['switchWords']) else: if os.path.isfile(switchFilename): os.remove(switchFilename) @@ -5094,8 +5132,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autotags.txt' if fields.get('autoTags'): - storeValue(autoTagsFilename, - fields['autoTags'], 'writeonly') + with open(autoTagsFilename, 'w+') as autoTagsFile: + autoTagsFile.write(fields['autoTags']) else: if os.path.isfile(autoTagsFilename): os.remove(autoTagsFilename) @@ -5106,8 +5144,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/autocw.txt' if fields.get('autoCW'): - storeValue(autoCWFilename, - fields['autoCW'], 'writeonly') + with open(autoCWFilename, 'w+') as autoCWFile: + autoCWFile.write(fields['autoCW']) else: if os.path.isfile(autoCWFilename): os.remove(autoCWFilename) @@ -5118,8 +5156,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/blocking.txt' if fields.get('blocked'): - storeValue(blockedFilename, - fields['blocked'], 'writeonly') + with open(blockedFilename, 'w+') as blockedfile: + blockedfile.write(fields['blocked']) else: if os.path.isfile(blockedFilename): os.remove(blockedFilename) @@ -5131,8 +5169,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/dmAllowedinstances.txt' if fields.get('dmAllowedInstances'): - storeValue(dmAllowedInstancesFilename, - fields['dmAllowedInstances'], 'writeonly') + with open(dmAllowedInstancesFilename, 'w+') as aFile: + aFile.write(fields['dmAllowedInstances']) else: if os.path.isfile(dmAllowedInstancesFilename): os.remove(dmAllowedInstancesFilename) @@ -5143,8 +5181,8 @@ class PubServer(BaseHTTPRequestHandler): baseDir + '/accounts/' + \ nickname + '@' + domain + '/allowedinstances.txt' if fields.get('allowedInstances'): - storeValue(allowedInstancesFilename, - fields['allowedInstances'], 'writeonly') + with open(allowedInstancesFilename, 'w+') as aFile: + aFile.write(fields['allowedInstances']) else: if os.path.isfile(allowedInstancesFilename): os.remove(allowedInstancesFilename) @@ -5159,8 +5197,8 @@ class PubServer(BaseHTTPRequestHandler): path.startswith('/users/' + adminNickname + '/'): self.server.peertubeInstances.clear() - storeValue(peertubeInstancesFile, - fields['ptInstances'], 'writeonly') + with open(peertubeInstancesFile, 'w+') as aFile: + aFile.write(fields['ptInstances']) ptInstancesList = \ fields['ptInstances'].split('\n') if ptInstancesList: @@ -5182,9 +5220,8 @@ class PubServer(BaseHTTPRequestHandler): nickname + '@' + domain + \ '/gitprojects.txt' if fields.get('gitProjects'): - projectsStr = fields['gitProjects'].lower() - storeValue(gitProjectsFilename, - projectsStr, 'writeonly') + with open(gitProjectsFilename, 'w+') as aFile: + aFile.write(fields['gitProjects'].lower()) else: if os.path.isfile(gitProjectsFilename): os.remove(gitProjectsFilename) @@ -13120,7 +13157,11 @@ class PubServer(BaseHTTPRequestHandler): with open(mediaFilename, 'rb') as avFile: mediaBinary = avFile.read() etag = sha1(mediaBinary).hexdigest() # nosec - storeValue(mediaTagFilename, etag, 'writeonly') + try: + with open(mediaTagFilename, 'w+') as etagFile: + etagFile.write(etag) + except BaseException: + pass mediaFileType = mediaFileMimeType(checkPath) self._set_headers_head(mediaFileType, fileLength, @@ -13285,8 +13326,13 @@ class PubServer(BaseHTTPRequestHandler): lastUsedFilename = \ self.server.baseDir + '/accounts/' + \ nickname + '@' + self.server.domain + '/.lastUsed' - lastUsedStr = str(int(time.time())) - storeValue(lastUsedFilename, lastUsedStr, 'writeonly') + try: + lastUsedFile = open(lastUsedFilename, 'w+') + if lastUsedFile: + lastUsedFile.write(str(int(time.time()))) + lastUsedFile.close() + except BaseException: + pass mentionsStr = '' if fields.get('mentions'): diff --git a/desktop_client.py b/desktop_client.py index ee7fccf83..ce47c52aa 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -56,7 +56,6 @@ from bookmarks import sendBookmarkViaServer from bookmarks import sendUndoBookmarkViaServer from delete import sendDeleteViaServer from person import getActorJson -from storage import storeValue def _desktopHelp() -> None: @@ -176,7 +175,10 @@ def _markPostAsRead(actor: str, postId: str, postCategory: str) -> None: except Exception as e: print('WARN: Failed to mark post as read' + str(e)) else: - storeValue(readPostsFilename, postId, 'write') + readFile = open(readPostsFilename, 'w+') + if readFile: + readFile.write(postId + '\n') + readFile.close() def _hasReadPost(actor: str, postId: str, postCategory: str) -> bool: diff --git a/epicyon.py b/epicyon.py index d4f839135..a36e4af93 100644 --- a/epicyon.py +++ b/epicyon.py @@ -88,7 +88,6 @@ from announce import sendAnnounceViaServer from socnet import instancesGraph from migrate import migrateAccounts from desktop_client import runDesktopClient -from storage import storeValue def str2bool(v) -> bool: @@ -760,8 +759,12 @@ if args.socnet: proxyType, args.port, httpPrefix, debug, __version__) - if storeValue('socnet.dot', dotGraph, 'writeonly'): - print('Saved to socnet.dot') + try: + with open('socnet.dot', 'w+') as fp: + fp.write(dotGraph) + print('Saved to socnet.dot') + except BaseException: + pass sys.exit() if args.postsraw: diff --git a/follow.py b/follow.py index 0d447cf8f..f7a48ed10 100644 --- a/follow.py +++ b/follow.py @@ -30,7 +30,6 @@ from webfinger import webfingerHandle from auth import createBasicAuthHeader from session import getJson from session import postJson -from storage import storeValue def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: @@ -65,7 +64,8 @@ def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: lastSeenDir + '/' + actor.replace('/', '#') + '.txt' print('lastSeenFilename: ' + lastSeenFilename) if not os.path.isfile(lastSeenFilename): - storeValue(lastSeenFilename, '100', 'writeonly') + with open(lastSeenFilename, 'w+') as fp: + fp.write(str(100)) break @@ -279,7 +279,8 @@ def unfollowAccount(baseDir: str, nickname: str, domain: str, with open(unfollowedFilename, "a+") as f: f.write(handleToUnfollow + '\n') else: - storeValue(unfollowedFilename, handleToUnfollow, 'write') + with open(unfollowedFilename, "w+") as f: + f.write(handleToUnfollow + '\n') return True @@ -606,7 +607,8 @@ def _storeFollowRequest(baseDir: str, print('DEBUG: ' + approveHandleStored + ' is already awaiting approval') else: - storeValue(approveFollowsFilename, approveHandleStored, 'write') + with open(approveFollowsFilename, "w+") as fp: + fp.write(approveHandleStored + '\n') # store the follow request in its own directory # We don't rely upon the inbox because items in there could expire @@ -763,7 +765,9 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, 'Failed to write entry to followers file ' + str(e)) else: - storeValue(followersFilename, approveHandle, 'write') + followersFile = open(followersFilename, "w+") + followersFile.write(approveHandle + '\n') + followersFile.close() print('Beginning follow accept') return followedAccountAccepts(session, baseDir, httpPrefix, diff --git a/followingCalendar.py b/followingCalendar.py index 3b4665dbe..650687e14 100644 --- a/followingCalendar.py +++ b/followingCalendar.py @@ -8,7 +8,6 @@ __status__ = "Production" __module_group__ = "Calendar" import os -from storage import storeValue def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, @@ -31,7 +30,8 @@ def receivingCalendarEvents(baseDir: str, nickname: str, domain: str, # create a new calendar file from the following file with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() - storeValue(calendarFilename, followingHandles, 'writeonly') + with open(calendarFilename, 'w+') as fp: + fp.write(followingHandles) return handle + '\n' in open(calendarFilename).read() @@ -75,7 +75,8 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, with open(followingFilename, 'r') as followingFile: followingHandles = followingFile.read() if add: - storeValue(calendarFilename, followingHandles + handle, 'write') + with open(calendarFilename, 'w+') as fp: + fp.write(followingHandles + handle + '\n') # already in the calendar file? if handle + '\n' in followingHandles: @@ -85,14 +86,16 @@ def _receiveCalendarEvents(baseDir: str, nickname: str, domain: str, return # remove from calendar file followingHandles = followingHandles.replace(handle + '\n', '') - storeValue(calendarFilename, followingHandles, 'writeonly') + with open(calendarFilename, 'w+') as fp: + fp.write(followingHandles) else: print(handle + ' not in followingCalendar.txt') # not already in the calendar file if add: # append to the list of handles followingHandles += handle + '\n' - storeValue(calendarFilename, followingHandles, 'writeonly') + with open(calendarFilename, 'w+') as fp: + fp.write(followingHandles) def addPersonToCalendar(baseDir: str, nickname: str, domain: str, diff --git a/git.py b/git.py index 6f66d9c3e..71fdc1442 100644 --- a/git.py +++ b/git.py @@ -9,7 +9,6 @@ __module_group__ = "ActivityPub" import os import html -from storage import storeValue def _gitFormatContent(content: str) -> str: @@ -212,10 +211,12 @@ def receiveGitPatch(baseDir: str, nickname: str, domain: str, return False patchStr = \ _gitAddFromHandle(patchStr, '@' + fromNickname + '@' + fromDomain) - if storeValue(patchFilename, patchStr, 'writeonly'): + with open(patchFilename, 'w+') as patchFile: + patchFile.write(patchStr) patchNotifyFilename = \ baseDir + '/accounts/' + \ nickname + '@' + domain + '/.newPatchContent' - if storeValue(patchNotifyFilename, patchStr, 'writeonly'): + with open(patchNotifyFilename, 'w+') as patchFile: + patchFile.write(patchStr) return True return False diff --git a/happening.py b/happening.py index 730b45162..f1443164a 100644 --- a/happening.py +++ b/happening.py @@ -16,7 +16,6 @@ from utils import isPublicPost from utils import loadJson from utils import saveJson from utils import locatePost -from storage import storeValue def _validUuid(testUuid: str, version=4): @@ -37,7 +36,12 @@ def _removeEventFromTimeline(eventId: str, tlEventsFilename: str) -> None: return with open(tlEventsFilename, 'r') as fp: eventsTimeline = fp.read().replace(eventId + '\n', '') - storeValue(tlEventsFilename, eventsTimeline, 'writeonly') + try: + with open(tlEventsFilename, 'w+') as fp2: + fp2.write(eventsTimeline) + except BaseException: + print('ERROR: unable to save events timeline') + pass def saveEventPost(baseDir: str, handle: str, postId: str, @@ -101,7 +105,9 @@ def saveEventPost(baseDir: str, handle: str, postId: str, tlEventsFilename + ' ' + str(e)) return False else: - storeValue(tlEventsFilename, eventId, 'write') + tlEventsFile = open(tlEventsFilename, 'w+') + tlEventsFile.write(eventId + '\n') + tlEventsFile.close() # create a directory for the calendar year if not os.path.isdir(calendarPath + '/' + str(eventYear)): @@ -128,16 +134,17 @@ def saveEventPost(baseDir: str, handle: str, postId: str, # a new event has been added calendarNotificationFilename = \ baseDir + '/accounts/' + handle + '/.newCalendar' - calEventStr = \ - '/calendar?year=' + \ - str(eventYear) + \ - '?month=' + \ - str(eventMonthNumber) + \ - '?day=' + \ - str(eventDayOfMonth) - if not storeValue(calendarNotificationFilename, - calEventStr, 'write'): + calendarNotificationFile = \ + open(calendarNotificationFilename, 'w+') + if not calendarNotificationFile: return False + calendarNotificationFile.write('/calendar?year=' + + str(eventYear) + + '?month=' + + str(eventMonthNumber) + + '?day=' + + str(eventDayOfMonth)) + calendarNotificationFile.close() return True diff --git a/inbox.py b/inbox.py index 0c24e586c..9e9751f26 100644 --- a/inbox.py +++ b/inbox.py @@ -83,7 +83,6 @@ from categories import guessHashtagCategory from context import hasValidContext from speaker import updateSpeaker from announce import isSelfAnnounce -from storage import storeValue def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -128,7 +127,10 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: daysSinceEpoch = daysDiff.days tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n' if not os.path.isfile(tagsFilename): - storeValue(tagsFilename, tagline, 'write') + tagsFile = open(tagsFilename, "w+") + if tagsFile: + tagsFile.write(tagline) + tagsFile.close() else: if postUrl not in open(tagsFilename).read(): try: @@ -1458,7 +1460,10 @@ def _receiveAnnounce(recentPostsCache: {}, postJsonObject, personCache, translate, lookupActor, themeName) - storeValue(postFilename + '.tts', '\n', 'writeonly') + ttsFile = open(postFilename + '.tts', "w+") + if ttsFile: + ttsFile.write('\n') + ttsFile.close() if debug: print('DEBUG: Obtaining actor for announce post ' + @@ -1637,9 +1642,15 @@ def populateReplies(baseDir: str, httpPrefix: str, domain: str, if numLines > maxReplies: return False if messageId not in open(postRepliesFilename).read(): - storeValue(postRepliesFilename, messageId, 'append') + repliesFile = open(postRepliesFilename, 'a+') + if repliesFile: + repliesFile.write(messageId + '\n') + repliesFile.close() else: - storeValue(postRepliesFilename, messageId, 'write') + repliesFile = open(postRepliesFilename, 'w+') + if repliesFile: + repliesFile.write(messageId + '\n') + repliesFile.close() return True @@ -1803,7 +1814,8 @@ def _dmNotify(baseDir: str, handle: str, url: str) -> None: return dmFile = accountDir + '/.newDM' if not os.path.isfile(dmFile): - storeValue(dmFile, url, 'writeonly') + with open(dmFile, 'w+') as fp: + fp.write(url) def _alreadyLiked(baseDir: str, nickname: str, domain: str, @@ -1883,8 +1895,20 @@ def _likeNotify(baseDir: str, domain: str, onionDomain: str, prevLikeStr = fp.read() if prevLikeStr == likeStr: return - storeValue(prevLikeFile, likeStr, 'writeonly') - storeValue(likeFile, likeStr, 'writeonly') + try: + with open(prevLikeFile, 'w+') as fp: + fp.write(likeStr) + except BaseException: + print('ERROR: unable to save previous like notification ' + + prevLikeFile) + pass + try: + with open(likeFile, 'w+') as fp: + fp.write(likeStr) + except BaseException: + print('ERROR: unable to write like notification file ' + + likeFile) + pass def _replyNotify(baseDir: str, handle: str, url: str) -> None: @@ -1895,7 +1919,8 @@ def _replyNotify(baseDir: str, handle: str, url: str) -> None: return replyFile = accountDir + '/.newReply' if not os.path.isfile(replyFile): - storeValue(replyFile, url, 'writeonly') + with open(replyFile, 'w+') as fp: + fp.write(url) def _gitPatchNotify(baseDir: str, handle: str, @@ -1909,7 +1934,8 @@ def _gitPatchNotify(baseDir: str, handle: str, patchFile = accountDir + '/.newPatch' subject = subject.replace('[PATCH]', '').strip() handle = '@' + fromNickname + '@' + fromDomain - storeValue(patchFile, 'git ' + handle + ' ' + subject, 'writeonly') + with open(patchFile, 'w+') as fp: + fp.write('git ' + handle + ' ' + subject) def _groupHandle(baseDir: str, handle: str) -> bool: @@ -2080,7 +2106,13 @@ def inboxUpdateIndex(boxname: str, baseDir: str, handle: str, except Exception as e: print('WARN: Failed to write entry to index ' + str(e)) else: - storeValue(indexFilename, destinationFilename, 'write') + try: + indexFile = open(indexFilename, 'w+') + if indexFile: + indexFile.write(destinationFilename + '\n') + indexFile.close() + except Exception as e: + print('WARN: Failed to write initial entry to index ' + str(e)) return False @@ -2113,8 +2145,8 @@ def _updateLastSeen(baseDir: str, handle: str, actor: str) -> None: if int(daysSinceEpochFile) == daysSinceEpoch: # value hasn't changed, so we can save writing anything to file return - daysSinceEpochStr = str(daysSinceEpoch) - storeValue(lastSeenFilename, daysSinceEpochStr, 'writeonly') + with open(lastSeenFilename, 'w+') as lastSeenFile: + lastSeenFile.write(str(daysSinceEpoch)) def _bounceDM(senderPostId: str, session, httpPrefix: str, @@ -2558,7 +2590,10 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, # This enables you to ignore a threat that's getting boring if isReplyToMutedPost: print('MUTE REPLY: ' + destinationFilename) - storeValue(destinationFilename + '.muted', '\n', 'writeonly') + muteFile = open(destinationFilename + '.muted', 'w+') + if muteFile: + muteFile.write('\n') + muteFile.close() # update the indexes for different timelines for boxname in updateIndexList: diff --git a/media.py b/media.py index 5530f60f5..8b83d4f0d 100644 --- a/media.py +++ b/media.py @@ -21,7 +21,6 @@ from shutil import copyfile from shutil import rmtree from shutil import move from city import spoofGeolocation -from storage import storeValue def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None: @@ -74,8 +73,11 @@ def _spoofMetaData(baseDir: str, nickname: str, domain: str, decoySeed = int(fp.read()) else: decoySeed = randint(10000, 10000000000000000) - decoySeedStr = str(decoySeed) - storeValue(decoySeedFilename, decoySeedStr, 'writeonly') + try: + with open(decoySeedFilename, 'w+') as fp: + fp.write(str(decoySeed)) + except BaseException: + pass if os.path.isfile('/usr/bin/exiftool'): print('Spoofing metadata in ' + outputFilename + ' using exiftool') @@ -190,7 +192,11 @@ def _updateEtag(mediaFilename: str) -> None: # calculate hash etag = sha1(data).hexdigest() # nosec # save the hash - storeValue(mediaFilename + '.etag', etag, 'writeonly') + try: + with open(mediaFilename + '.etag', 'w+') as etagFile: + etagFile.write(etag) + except BaseException: + pass def attachMedia(baseDir: str, httpPrefix: str, diff --git a/migrate.py b/migrate.py index 9b03857fa..4cca4b0e8 100644 --- a/migrate.py +++ b/migrate.py @@ -15,7 +15,6 @@ from blocking import isBlocked from session import getJson from posts import getUserUrl from follow import unfollowAccount -from storage import storeValue def _moveFollowingHandlesForAccount(baseDir: str, nickname: str, domain: str, @@ -149,11 +148,11 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, # save the new handles to the refollow list if os.path.isfile(refollowFilename): - storeValue(refollowFilename, - movedToHandle, 'append') + with open(refollowFilename, 'a+') as f: + f.write(movedToHandle + '\n') else: - storeValue(refollowFilename, - movedToHandle, 'write') + with open(refollowFilename, 'w+') as f: + f.write(movedToHandle + '\n') followersFilename = \ baseDir + '/accounts/' + nickname + '@' + domain + '/followers.txt' diff --git a/newsdaemon.py b/newsdaemon.py index c816417dd..bc8e4f5f0 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -34,7 +34,6 @@ from utils import clearFromPostCaches from utils import dangerousMarkup from inbox import storeHashTags from session import createSession -from storage import storeValue def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: @@ -56,13 +55,19 @@ def _updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: print('WARN: Failed to write entry to feeds posts index ' + indexFilename + ' ' + str(e)) else: - storeValue(indexFilename, postId, 'write') + feedsFile = open(indexFilename, 'w+') + if feedsFile: + feedsFile.write(postId + '\n') + feedsFile.close() def _saveArrivedTime(baseDir: str, postFilename: str, arrived: str) -> None: """Saves the time when an rss post arrived to a file """ - storeValue(postFilename + '.arrived', arrived, 'writeonly') + arrivedFile = open(postFilename + '.arrived', 'w+') + if arrivedFile: + arrivedFile.write(arrived) + arrivedFile.close() def _removeControlCharacters(content: str) -> str: @@ -404,7 +409,8 @@ def _createNewsMirror(baseDir: str, domain: str, for removePostId in removals: indexContent = \ indexContent.replace(removePostId + '\n', '') - storeValue(mirrorIndexFilename, indexContent, 'writeonly') + with open(mirrorIndexFilename, "w+") as indexFile: + indexFile.write(indexContent) mirrorArticleDir = mirrorDir + '/' + postIdNumber if os.path.isdir(mirrorArticleDir): @@ -429,9 +435,15 @@ def _createNewsMirror(baseDir: str, domain: str, # append the post Id number to the index file if os.path.isfile(mirrorIndexFilename): - storeValue(mirrorIndexFilename, postIdNumber, 'append') + indexFile = open(mirrorIndexFilename, "a+") + if indexFile: + indexFile.write(postIdNumber + '\n') + indexFile.close() else: - storeValue(mirrorIndexFilename, postIdNumber, 'write') + indexFile = open(mirrorIndexFilename, "w+") + if indexFile: + indexFile.write(postIdNumber + '\n') + indexFile.close() return True diff --git a/person.py b/person.py index 9cb0af29b..0771bd115 100644 --- a/person.py +++ b/person.py @@ -52,7 +52,6 @@ from session import createSession from session import getJson from webfinger import webfingerHandle from pprint import pprint -from storage import storeValue def generateRSAKey() -> (str, str): @@ -495,13 +494,15 @@ def createPerson(baseDir: str, nickname: str, domain: str, port: int, if manualFollowerApproval: followDMsFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.followDMs' - storeValue(followDMsFilename, '\n', 'writeonly') + with open(followDMsFilename, 'w+') as fFile: + fFile.write('\n') # notify when posts are liked if nickname != 'news': notifyLikesFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/.notifyLikes' - storeValue(notifyLikesFilename, '\n', 'writeonly') + with open(notifyLikesFilename, 'w+') as nFile: + nFile.write('\n') theme = getConfigParam(baseDir, 'theme') if not theme: @@ -922,9 +923,15 @@ def suspendAccount(baseDir: str, nickname: str, domain: str) -> None: for suspended in lines: if suspended.strip('\n').strip('\r') == nickname: return - storeValue(suspendedFilename, nickname, 'append') + suspendedFile = open(suspendedFilename, 'a+') + if suspendedFile: + suspendedFile.write(nickname + '\n') + suspendedFile.close() else: - storeValue(suspendedFilename, nickname, 'write') + suspendedFile = open(suspendedFilename, 'w+') + if suspendedFile: + suspendedFile.write(nickname + '\n') + suspendedFile.close() def canRemovePost(baseDir: str, nickname: str, @@ -1125,7 +1132,10 @@ def isPersonSnoozed(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - storeValue(snoozedFilename, content, 'writeonly') + writeSnoozedFile = open(snoozedFilename, 'w+') + if writeSnoozedFile: + writeSnoozedFile.write(content) + writeSnoozedFile.close() if snoozeActor + ' ' in open(snoozedFilename).read(): return True @@ -1175,7 +1185,10 @@ def personUnsnooze(baseDir: str, nickname: str, domain: str, with open(snoozedFilename, 'r') as snoozedFile: content = snoozedFile.read().replace(replaceStr, '') if content: - storeValue(snoozedFilename, content, 'writeonly') + writeSnoozedFile = open(snoozedFilename, 'w+') + if writeSnoozedFile: + writeSnoozedFile.write(content) + writeSnoozedFile.close() def setPersonNotes(baseDir: str, nickname: str, domain: str, @@ -1191,7 +1204,8 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, if not os.path.isdir(notesDir): os.mkdir(notesDir) notesFilename = notesDir + '/' + handle + '.txt' - storeValue(notesFilename, notes, 'writeonly') + with open(notesFilename, 'w+') as notesFile: + notesFile.write(notes) return True diff --git a/petnames.py b/petnames.py index 52ece3c3f..21c46095a 100644 --- a/petnames.py +++ b/petnames.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os -from storage import storeValue def setPetName(baseDir: str, nickname: str, domain: str, @@ -41,14 +40,17 @@ def setPetName(baseDir: str, nickname: str, domain: str, else: newPetnamesStr += entry # save the updated petnames file - storeValue(petnamesFilename, newPetnamesStr, 'writeonly') + with open(petnamesFilename, 'w+') as petnamesFile: + petnamesFile.write(newPetnamesStr) return True # entry does not exist in the petnames file - storeValue(petnamesFilename, entry, 'append') + with open(petnamesFilename, 'a+') as petnamesFile: + petnamesFile.write(entry) return True # first entry - storeValue(petnamesFilename, entry, 'writeonly') + with open(petnamesFilename, 'w+') as petnamesFile: + petnamesFile.write(entry) return True diff --git a/posts.py b/posts.py index 05291fedf..6e275d45f 100644 --- a/posts.py +++ b/posts.py @@ -71,7 +71,6 @@ from filters import isFiltered from git import convertPostToPatch from linked_data_sig import generateJsonSignature from petnames import resolvePetnames -from storage import storeValue def isModerator(baseDir: str, nickname: str) -> bool: @@ -734,7 +733,17 @@ def _updateHashtagsIndex(baseDir: str, tag: {}, newPostId: str) -> None: tagsFile.write(tagline) tagsFile.close() else: - storeValue(tagsFilename, tagline, 'prepend') + # prepend to tags index file + if tagline not in open(tagsFilename).read(): + try: + with open(tagsFilename, 'r+') as tagsFile: + content = tagsFile.read() + if tagline not in content: + tagsFile.seek(0, 0) + tagsFile.write(tagline + content) + except Exception as e: + print('WARN: Failed to write entry to tags file ' + + tagsFilename + ' ' + str(e)) def _addSchedulePost(baseDir: str, nickname: str, domain: str, @@ -758,7 +767,10 @@ def _addSchedulePost(baseDir: str, nickname: str, domain: str, print('WARN: Failed to write entry to scheduled posts index ' + scheduleIndexFilename + ' ' + str(e)) else: - storeValue(scheduleIndexFilename, indexStr, 'write') + scheduleFile = open(scheduleIndexFilename, 'w+') + if scheduleFile: + scheduleFile.write(indexStr + '\n') + scheduleFile.close() def _appendEventFields(newPost: {}, @@ -1182,7 +1194,10 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, newPost['moderationStatus'] = 'pending' # save to index file moderationIndexFile = baseDir + '/accounts/moderation.txt' - storeValue(moderationIndexFile, newPostId, 'append') + modFile = open(moderationIndexFile, "a+") + if modFile: + modFile.write(newPostId + '\n') + modFile.close() # If a patch has been posted - i.e. the output from # git format-patch - then convert the activitypub type @@ -1290,7 +1305,10 @@ def pinPost(baseDir: str, nickname: str, domain: str, """ accountDir = baseDir + '/accounts/' + nickname + '@' + domain pinnedFilename = accountDir + '/pinToProfile.txt' - storeValue(pinnedFilename, pinnedContent, 'writeonly') + pinFile = open(pinnedFilename, "w+") + if pinFile: + pinFile.write(pinnedContent) + pinFile.close() def undoPinnedPost(baseDir: str, nickname: str, domain: str) -> None: @@ -1832,7 +1850,11 @@ def createReportPost(baseDir: str, newReportFile = baseDir + '/accounts/' + handle + '/.newReport' if os.path.isfile(newReportFile): continue - storeValue(newReportFile, toUrl + '/moderation', 'writeonly') + try: + with open(newReportFile, 'w+') as fp: + fp.write(toUrl + '/moderation') + except BaseException: + pass return postJsonObject @@ -1876,7 +1898,8 @@ def threadSendPost(session, postJsonStr: str, federationList: [], if debug: # save the log file postLogFilename = baseDir + '/post.log' - storeValue(postLogFilename, logStr, 'append') + with open(postLogFilename, "a+") as logFile: + logFile.write(logStr + '\n') if postResult: if debug: @@ -3429,7 +3452,10 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str, break # save the new index file if len(newIndex) > 0: - storeValue(indexFilename, newIndex, 'writeonly') + indexFile = open(indexFilename, 'w+') + if indexFile: + indexFile.write(newIndex) + indexFile.close() postsInBoxDict = {} postsCtr = 0 @@ -3812,7 +3838,8 @@ def checkDomains(session, baseDir: str, updateFollowerWarnings = True if updateFollowerWarnings and followerWarningStr: - storeValue(followerWarningFilename, followerWarningStr, 'writeonly') + with open(followerWarningFilename, 'w+') as fp: + fp.write(followerWarningStr) if not singleCheck: print(followerWarningStr) @@ -3892,7 +3919,10 @@ def _rejectAnnounce(announceFilename: str, # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): - storeValue(announceFilename + '.reject', '\n', 'writeonly') + rejectAnnounceFile = open(announceFilename + '.reject', "w+") + if rejectAnnounceFile: + rejectAnnounceFile.write('\n') + rejectAnnounceFile.close() def downloadAnnounce(session, baseDir: str, httpPrefix: str, diff --git a/question.py b/question.py index 5d0e08fc5..7b7328f44 100644 --- a/question.py +++ b/question.py @@ -11,7 +11,6 @@ import os from utils import locatePost from utils import loadJson from utils import saveJson -from storage import storeValue def questionUpdateVotes(baseDir: str, nickname: str, domain: str, @@ -68,17 +67,21 @@ def questionUpdateVotes(baseDir: str, nickname: str, domain: str, votersFilename = questionPostFilename.replace('.json', '.voters') if not os.path.isfile(votersFilename): # create a new voters file - vStr = replyJson['actor'] + \ - votersFileSeparator + \ - foundAnswer - storeValue(votersFilename, vStr, 'write') + votersFile = open(votersFilename, 'w+') + if votersFile: + votersFile.write(replyJson['actor'] + + votersFileSeparator + + foundAnswer + '\n') + votersFile.close() else: if replyJson['actor'] not in open(votersFilename).read(): # append to the voters file - vStr = replyJson['actor'] + \ - votersFileSeparator + \ - foundAnswer - storeValue(votersFilename, vStr, 'append') + votersFile = open(votersFilename, "a+") + if votersFile: + votersFile.write(replyJson['actor'] + + votersFileSeparator + + foundAnswer + '\n') + votersFile.close() else: # change an entry in the voters file with open(votersFilename, "r") as votersFile: diff --git a/shares.py b/shares.py index f3f1d2adf..4fb62daad 100644 --- a/shares.py +++ b/shares.py @@ -20,7 +20,6 @@ from utils import loadJson from utils import saveJson from utils import getImageExtensions from media import processMetaData -from storage import storeValue def getValidSharedItemID(displayName: str) -> str: @@ -162,10 +161,12 @@ def addShare(baseDir: str, newShareFile = accountDir + '/.newShare' if not os.path.isfile(newShareFile): nickname = handle.split('@')[0] - storeValue(newShareFile, - httpPrefix + '://' + domainFull + - '/users/' + nickname + '/tlshares', - 'writeonly') + try: + with open(newShareFile, 'w+') as fp: + fp.write(httpPrefix + '://' + domainFull + + '/users/' + nickname + '/tlshares') + except BaseException: + pass break diff --git a/storage.py b/storage.py deleted file mode 100644 index 0417796f0..000000000 --- a/storage.py +++ /dev/null @@ -1,56 +0,0 @@ -__filename__ = "storage.py" -__author__ = "Bob Mottram" -__license__ = "AGPL3+" -__version__ = "1.2.0" -__maintainer__ = "Bob Mottram" -__email__ = "bob@freedombone.net" -__status__ = "Production" -__module_group__ = "storage" - -import os - - -def storeValue(filename: str, lineStr: str, storeType: str) -> bool: - """Stores a line to a file - """ - if not lineStr.endswith('\n'): - if storeType != 'writeonly': - lineStr += '\n' - - if storeType[0] == 'a': - if not os.path.isfile(filename): - storeType = 'write' - - if storeType[0] == 'a': - if not os.path.isfile(filename): - return False - # append - try: - with open(filename, "a+") as fp: - fp.write(lineStr) - return True - except Exception as e: - print('ERROR: unable to append to ' + filename + ' ' + str(e)) - pass - elif storeType[0] == 'w': - # new file - try: - with open(filename, "w+") as fp: - fp.write(lineStr) - return True - except Exception as e: - print('ERROR: unable to write to ' + filename + ' ' + str(e)) - pass - elif storeType[0] == 'p': - # prepend - if lineStr not in open(filename).read(): - try: - with open(filename, 'r+') as fp: - content = fp.read() - if lineStr not in content: - fp.seek(0, 0) - fp.write(lineStr + content) - except Exception as e: - print('WARN: Unable to prepend to ' + - filename + ' ' + str(e)) - return False diff --git a/tests.py b/tests.py index 8b18368f4..f270ae9e1 100644 --- a/tests.py +++ b/tests.py @@ -117,7 +117,6 @@ from mastoapiv1 import getNicknameFromMastoApiV1Id from webapp_post import prepareHtmlPostNickname from webapp_utils import markdownToHtml from speaker import speakerReplaceLinks -from storage import storeValue testServerAliceRunning = False testServerBobRunning = False @@ -3168,11 +3167,12 @@ def _testFunctions(): callGraphStr += ' }\n' clusterCtr += 1 callGraphStr += '\n}\n' - assert storeValue('epicyon_modules.dot', callGraphStr, 'writeonly') - print('Modules call graph saved to epicyon_modules.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + - '-Gsep=+100 -Tx11 epicyon_modules.dot') + with open('epicyon_modules.dot', 'w+') as fp: + fp.write(callGraphStr) + print('Modules call graph saved to epicyon_modules.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + + '-Gsep=+100 -Tx11 epicyon_modules.dot') callGraphStr = 'digraph Epicyon {\n\n' callGraphStr += ' size="8,6"; ratio=fill;\n' @@ -3223,11 +3223,12 @@ def _testFunctions(): '" [color=' + modColor + '];\n' callGraphStr += '\n}\n' - assert storeValue('epicyon.dot', callGraphStr, 'writeonly') - print('Call graph saved to epicyon.dot') - print('Plot using: ' + - 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + - '-Gsep=+120 -Tx11 epicyon.dot') + with open('epicyon.dot', 'w+') as fp: + fp.write(callGraphStr) + print('Call graph saved to epicyon.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + + '-Gsep=+120 -Tx11 epicyon.dot') def _testLinksWithinPost() -> None: @@ -3882,7 +3883,10 @@ def _testSpoofGeolocation() -> None: kmlStr += '\n' kmlStr += '' - assert storeValue('unittest_decoy.kml', kmlStr, 'writeonly') + kmlFile = open('unittest_decoy.kml', 'w+') + if kmlFile: + kmlFile.write(kmlStr) + kmlFile.close() def _testSkills() -> None: diff --git a/theme.py b/theme.py index 5fa2bac01..7fdb0d720 100644 --- a/theme.py +++ b/theme.py @@ -16,7 +16,6 @@ from shutil import make_archive from shutil import unpack_archive from shutil import rmtree from content import dangerousCSS -from storage import storeValue def importTheme(baseDir: str, filename: str) -> bool: @@ -362,7 +361,8 @@ def _setThemeFromDict(baseDir: str, name: str, continue css = setCSSparam(css, paramName, paramValue) filename = baseDir + '/' + filename - storeValue(filename, css, 'writeonly') + with open(filename, 'w+') as cssfile: + cssfile.write(css) if bgParams.get('login'): _setBackgroundFormat(baseDir, name, 'login', bgParams['login']) @@ -388,7 +388,8 @@ def _setBackgroundFormat(baseDir: str, name: str, with open(cssFilename, 'r') as cssfile: css = cssfile.read() css = css.replace('background.jpg', 'background.' + extension) - storeValue(cssFilename, css, 'writeonly') + with open(cssFilename, 'w+') as cssfile2: + cssfile2.write(css) def enableGrayscale(baseDir: str) -> None: @@ -406,10 +407,12 @@ def enableGrayscale(baseDir: str) -> None: css.replace('body, html {', 'body, html {\n filter: grayscale(100%);') filename = baseDir + '/' + filename - storeValue(filename, css, 'writeonly') + with open(filename, 'w+') as cssfile: + cssfile.write(css) grayscaleFilename = baseDir + '/accounts/.grayscale' if not os.path.isfile(grayscaleFilename): - storeValue(grayscaleFilename, ' ', 'writeonly') + with open(grayscaleFilename, 'w+') as grayfile: + grayfile.write(' ') def disableGrayscale(baseDir: str) -> None: @@ -426,7 +429,8 @@ def disableGrayscale(baseDir: str) -> None: css = \ css.replace('\n filter: grayscale(100%);', '') filename = baseDir + '/' + filename - storeValue(filename, css, 'writeonly') + with open(filename, 'w+') as cssfile: + cssfile.write(css) grayscaleFilename = baseDir + '/accounts/.grayscale' if os.path.isfile(grayscaleFilename): os.remove(grayscaleFilename) @@ -466,7 +470,8 @@ def _setCustomFont(baseDir: str): customFontType + "')") css = setCSSparam(css, "*font-family", "'CustomFont'") filename = baseDir + '/' + filename - storeValue(filename, css, 'writeonly') + with open(filename, 'w+') as cssfile: + cssfile.write(css) def _readVariablesFile(baseDir: str, themeName: str, @@ -734,7 +739,8 @@ def _setClearCacheFlag(baseDir: str) -> None: if not os.path.isdir(baseDir + '/accounts'): return flagFilename = baseDir + '/accounts/.clear_cache' - storeValue(flagFilename, '\n', 'writeonly') + with open(flagFilename, 'w+') as flagFile: + flagFile.write('\n') def setTheme(baseDir: str, name: str, domain: str, diff --git a/utils.py b/utils.py index e22a2f68d..fd159e8af 100644 --- a/utils.py +++ b/utils.py @@ -18,7 +18,6 @@ from pprint import pprint from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes -from storage import storeValue # posts containing these strings will always get screened out, # both incoming and outgoing. @@ -44,7 +43,9 @@ def refreshNewswire(baseDir: str): refreshNewswireFilename = baseDir + '/accounts/.refresh_newswire' if os.path.isfile(refreshNewswireFilename): return - storeValue(refreshNewswireFilename, '\n', 'writeonly') + refreshFile = open(refreshNewswireFilename, 'w+') + refreshFile.write('\n') + refreshFile.close() def getSHA256(msg: str): @@ -489,13 +490,15 @@ def saveJson(jsonObject: {}, filename: str) -> bool: """Saves json to a file """ tries = 0 - storeStr = json.dumps(jsonObject) while tries < 5: - if storeValue(filename, storeStr, 'writeonly'): - return True - print('WARN: saveJson ' + str(tries)) - time.sleep(1) - tries += 1 + try: + with open(filename, 'w+') as fp: + fp.write(json.dumps(jsonObject)) + return True + except BaseException: + print('WARN: saveJson ' + str(tries)) + time.sleep(1) + tries += 1 return False @@ -939,7 +942,8 @@ def _setDefaultPetName(baseDir: str, nickname: str, domain: str, followNickname + '@' + followDomain + '\n' if not os.path.isfile(petnamesFilename): # if there is no existing petnames lookup file - storeValue(petnamesFilename, petnameLookupEntry, 'writeonly') + with open(petnamesFilename, 'w+') as petnamesFile: + petnamesFile.write(petnameLookupEntry) return with open(petnamesFilename, 'r') as petnamesFile: @@ -996,7 +1000,8 @@ def followPerson(baseDir: str, nickname: str, domain: str, for line in lines: if handleToFollow not in line: newLines += line - storeValue(unfollowedFilename, newLines, 'writeonly') + with open(unfollowedFilename, 'w+') as f: + f.write(newLines) if not os.path.isdir(baseDir + '/accounts'): os.mkdir(baseDir + '/accounts') @@ -1024,7 +1029,8 @@ def followPerson(baseDir: str, nickname: str, domain: str, print('DEBUG: ' + handle + ' creating new following file to follow ' + handleToFollow + ', filename is ' + filename) - storeValue(filename, handleToFollow, 'write') + with open(filename, 'w+') as f: + f.write(handleToFollow + '\n') if followFile.endswith('following.txt'): # Default to adding new follows to the calendar. @@ -1346,7 +1352,8 @@ def deletePost(baseDir: str, httpPrefix: str, # hashtag file os.remove(tagIndexFilename) else: - storeValue(tagIndexFilename, newlines, 'writeonly') + with open(tagIndexFilename, "w+") as f: + f.write(newlines) # remove any replies repliesFilename = postFilename.replace('.json', '.replies') @@ -2191,7 +2198,10 @@ def rejectPostId(baseDir: str, nickname: str, domain: str, if recentPostsCache['html'].get(postUrl): del recentPostsCache['html'][postUrl] - storeValue(postFilename + '.reject', '\n', 'writeonly') + rejectFile = open(postFilename + '.reject', "w+") + if rejectFile: + rejectFile.write('\n') + rejectFile.close() def isDM(postJsonObject: {}) -> bool: diff --git a/webapp_post.py b/webapp_post.py index e96689be2..f1f845b73 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -69,7 +69,6 @@ from webapp_question import insertQuestion from devices import E2EEdecryptMessageFromDevice from webfinger import webfingerHandle from speaker import updateSpeaker -from storage import storeValue def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None: @@ -157,7 +156,13 @@ def _saveIndividualPostAsHtmlToCache(baseDir: str, if not os.path.isdir(htmlPostCacheDir): os.mkdir(htmlPostCacheDir) - return storeValue(cachedPostFilename, postHtml, 'writeonly') + try: + with open(cachedPostFilename, 'w+') as fp: + fp.write(postHtml) + return True + except Exception as e: + print('ERROR: saving post to cache ' + str(e)) + return False def _getPostFromRecentCache(session, @@ -1327,8 +1332,10 @@ def individualPostAsHtml(allowDownloads: bool, postJsonObject, personCache, translate, postJsonObject['actor'], themeName) - storeValue(announceFilename + '.tts', - '\n', 'writeonly') + ttsFile = open(announceFilename + '.tts', "w+") + if ttsFile: + ttsFile.write('\n') + ttsFile.close() isAnnounced = True diff --git a/webapp_utils.py b/webapp_utils.py index 1c611473d..7797d040c 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -20,7 +20,6 @@ from cache import getPersonFromCache from cache import storePersonInCache from content import addHtmlTags from content import replaceEmojiFromTags -from storage import storeValue def _markdownEmphasisHtml(markdown: str) -> str: @@ -1388,4 +1387,5 @@ def setMinimal(baseDir: str, domain: str, nickname: str, if minimal and minimalFileExists: os.remove(minimalFilename) elif not minimal and not minimalFileExists: - storeValue(minimalFilename, '\n', 'writeonly') + with open(minimalFilename, 'w+') as fp: + fp.write('\n') diff --git a/webapp_welcome.py b/webapp_welcome.py index 9bd0f5111..78e1cf441 100644 --- a/webapp_welcome.py +++ b/webapp_welcome.py @@ -14,7 +14,6 @@ from utils import removeHtml from webapp_utils import htmlHeaderWithExternalStyle from webapp_utils import htmlFooter from webapp_utils import markdownToHtml -from storage import storeValue def isWelcomeScreenComplete(baseDir: str, nickname: str, domain: str) -> bool: @@ -35,7 +34,10 @@ def welcomeScreenIsComplete(baseDir: str, if not os.path.isdir(accountPath): return completeFilename = accountPath + '/.welcome_complete' - storeValue(completeFilename, '\n', 'writeonly') + completeFile = open(completeFilename, 'w+') + if completeFile: + completeFile.write('\n') + completeFile.close() def htmlWelcomeScreen(baseDir: str, nickname: str,