diff --git a/announce.py b/announce.py index 744cf87c..7f3cb496 100644 --- a/announce.py +++ b/announce.py @@ -18,6 +18,8 @@ from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost from utils import getCachedPostFilename +from utils import loadJson +from utils import saveJson from posts import sendSignedJson from posts import getPersonBox from session import postJson @@ -70,17 +72,7 @@ def undoAnnounceCollectionEntry(baseDir: str,postFilename: str,actor: str,domain collection within a post. Note that the "shares" collection has no relation to shared items in shares.py. It's shares of posts, not shares of physical objects. """ - postJsonObject=None - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception undoAnnounceCollectionEntry - '+str(e)) - time.sleep(2) - tries+=1 + postJsonObject=loadJson(postFilename) if postJsonObject: # remove any cached version of this announce so that the like icon is changed nickname=getNicknameFromActor(actor) @@ -122,33 +114,14 @@ def undoAnnounceCollectionEntry(baseDir: str,postFilename: str,actor: str,domain del postJsonObject['object']['shares'] else: postJsonObject['object']['shares']['totalItems']=len(postJsonObject['object']['shares']['items']) - tries=0 - while tries<5: - try: - with open(postFilename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(postJsonObject,postFilename) def updateAnnounceCollection(baseDir: str,postFilename: str,actor: str,domain: str,debug: bool) -> None: """Updates the announcements collection within a post Confusingly this is known as "shares", but isn't the same as shared items within shares.py It's shares of posts, not shares of physical objects. """ - postJsonObject=None - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception updateAnnounceCollection - '+str(e)) - time.sleep(1) - tries+=1 + postJsonObject=loadJson(postFilename) if postJsonObject: # remove any cached version of this announce so that the like icon is changed nickname=getNicknameFromActor(actor) @@ -197,16 +170,7 @@ def updateAnnounceCollection(baseDir: str,postFilename: str,actor: str,domain: s if debug: print('DEBUG: saving post with shares (announcements) added') pprint(postJsonObject) - tries=0 - while tries<5: - try: - with open(postFilename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(postJsonObject,postFilename) def announcedByPerson(postJsonObject: {}, nickname: str,domain: str) -> bool: """Returns True if the given post is announced by the given person @@ -269,16 +233,7 @@ def createAnnounce(session,baseDir: str,federationList: [], \ if saveToFile: outboxDir = createOutboxDir(nickname,domain,baseDir) filename=outboxDir+'/'+newAnnounceId.replace('/','#')+'.json' - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(newAnnounce, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(newAnnounce,filename) announceNickname=None announceDomain=None diff --git a/availability.py b/availability.py index 46eef175..e2018842 100644 --- a/availability.py +++ b/availability.py @@ -16,6 +16,8 @@ from posts import getPersonBox from session import postJson from utils import getNicknameFromActor from utils import getDomainFromActor +from utils import loadJson +from utils import saveJson def setAvailability(baseDir: str,nickname: str,domain: str, \ status: str) -> bool: @@ -27,29 +29,10 @@ def setAvailability(baseDir: str,nickname: str,domain: str, \ actorFilename=baseDir+'/accounts/'+nickname+'@'+domain+'.json' if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setAvailability - '+str(e)) - time.sleep(1) - tries+=1 + actorJson=loadJson(actorFilename) if actorJson: actorJson['availability']=status - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) return True def getAvailability(baseDir: str,nickname: str,domain: str) -> str: @@ -58,17 +41,7 @@ def getAvailability(baseDir: str,nickname: str,domain: str) -> str: actorFilename=baseDir+'/accounts/'+nickname+'@'+domain+'.json' if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getAvailability - '+str(e)) - time.sleep(1) - tries+=1 + actorJson=loadJson(actorFilename) if actorJson: if not actorJson.get('availability'): return None diff --git a/cache.py b/cache.py index 5694fe4e..12c933ba 100644 --- a/cache.py +++ b/cache.py @@ -10,6 +10,8 @@ import os import time import datetime import commentjson +from utils import loadJson +from utils import saveJson def storePersonInCache(baseDir: str,personUrl: str,personJson: {},personCache: {}) -> None: """Store an actor in the cache @@ -26,16 +28,7 @@ def storePersonInCache(baseDir: str,personUrl: str,personJson: {},personCache: { if os.path.isdir(baseDir+'/cache/actors'): cacheFilename=baseDir+'/cache/actors/'+personUrl.replace('/','#')+'.json' if not os.path.isfile(cacheFilename): - tries=0 - while tries<5: - try: - with open(cacheFilename, 'w') as fp: - commentjson.dump(personJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(personJson,cacheFilename) def getPersonFromCache(baseDir: str,personUrl: str,personCache: {}) -> {}: """Get an actor from the cache @@ -45,19 +38,7 @@ def getPersonFromCache(baseDir: str,personUrl: str,personCache: {}) -> {}: if not personCache.get(personUrl): cacheFilename=baseDir+'/cache/actors/'+personUrl.replace('/','#')+'.json' if os.path.isfile(cacheFilename): - personJson=None - tries=0 - while tries<5: - try: - with open(cacheFilename, 'r') as fp: - personJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getPersonFromCache - '+str(e)) - print('ERROR: unable to load actor from cache '+cacheFilename+' try '+str(tries)) - print(e) - time.sleep(1) - tries+=1 + personJson=loadJson(cacheFilename) if personJson: storePersonInCache(baseDir,personUrl,personJson,personCache) loadedFromFile=True diff --git a/capabilities.py b/capabilities.py index 03f2340f..e1845351 100644 --- a/capabilities.py +++ b/capabilities.py @@ -14,6 +14,8 @@ import commentjson from auth import createPassword from utils import getNicknameFromActor from utils import getDomainFromActor +from utils import loadJson +from utils import saveJson def getOcapFilename(baseDir :str,nickname: str,domain: str,actor :str,subdir: str) -> str: """Returns the filename for a particular capability accepted or granted @@ -127,16 +129,7 @@ def capabilitiesAccept(baseDir: str,httpPrefix: str, \ # if the capability already exists then load it from file if os.path.isfile(ocapFilename): - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - ocapAccept=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception capabilitiesAccept - '+str(e)) - time.sleep(1) - tries+=1 + ocapAccept=loadJson(ocapFilename) # otherwise create a new capability if not ocapAccept: acceptedActorNickname=getNicknameFromActor(acceptedActor) @@ -160,16 +153,7 @@ def capabilitiesAccept(baseDir: str,httpPrefix: str, \ ocapAccept['actor']=httpPrefix+"://"+fullDomain+'/users/'+nickname if saveToFile: - tries=0 - while tries<5: - try: - with open(ocapFilename, 'w') as fp: - commentjson.dump(ocapAccept, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(ocapAccept,ocapFilename) return ocapAccept def capabilitiesGrantedSave(baseDir :str,nickname :str,domain :str,ocap: {}) -> bool: @@ -181,16 +165,7 @@ def capabilitiesGrantedSave(baseDir :str,nickname :str,domain :str,ocap: {}) -> ocapFilename=getOcapFilename(baseDir,nickname,domain,ocap['actor'],'granted') if not ocapFilename: return False - tries=0 - while tries<5: - try: - with open(ocapFilename, 'w') as fp: - commentjson.dump(ocap, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(ocap,ocapFilename) return True def capabilitiesUpdate(baseDir: str,httpPrefix: str, \ @@ -232,16 +207,7 @@ def capabilitiesUpdate(baseDir: str,httpPrefix: str, \ } # read the existing capability - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - ocapJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception capabilitiesUpdate - '+str(e)) - time.sleep(1) - tries+=1 + ocapJson=loadJson(ocapFilename) # set the new capabilities list. eg. ["inbox:write","objects:read"] ocapJson['capability']=updateCaps @@ -260,16 +226,7 @@ def capabilitiesUpdate(baseDir: str,httpPrefix: str, \ ocapUpdate['object']=ocapJson # save it again - tries=0 - while tries<5: - try: - with open(ocapFilename, 'w') as fp: - commentjson.dump(ocapJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(ocapJson,ocapFilename) return ocapUpdate @@ -291,30 +248,11 @@ def capabilitiesReceiveUpdate(baseDir :str, \ print(ocapFilename) return False - ocapJson=None - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - ocapJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception capabilitiesReceiveUpdate - '+str(e)) - time.sleep(1) - tries+=1 + ocapJson=loadJson(ocapFilename) if ocapJson: ocapJson['id']=newCapabilitiesId ocapJson['capability']=capabilityList - tries=0 - while tries<5: - try: - with open(ocapFilename, 'w') as fp: - commentjson.dump(ocapJson, fp, indent=2, sort_keys=False) - return True - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + return saveJson(ocapJson,ocapFilename) return False diff --git a/config.py b/config.py index 413f2d4a..7cbf7644 100644 --- a/config.py +++ b/config.py @@ -10,6 +10,8 @@ import os import time import json import commentjson +from utils import loadJson +from utils import saveJson def createConfig(baseDir: str) -> None: """Creates a configuration file @@ -19,59 +21,26 @@ def createConfig(baseDir: str) -> None: return configJson = { } - tries=0 - while tries<5: - try: - with open(configFilename, 'w') as fp: - commentjson.dump(configJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(configJson,configFilename) def setConfigParam(baseDir: str, variableName: str, variableValue) -> None: """Sets a configuration value """ createConfig(baseDir) configFilename=baseDir+'/config.json' - tries=0 - while tries<5: - try: - with open(configFilename, 'r') as fp: - configJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setConfigParam - '+str(e)) - time.sleep(1) - tries+=1 + configJson={} + if os.path.isfile(configFilename): + configJson=loadJson(configFilename) configJson[variableName]=variableValue - tries=0 - while tries<5: - try: - with open(configFilename, 'w') as fp: - commentjson.dump(configJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(configJson,configFilename) def getConfigParam(baseDir: str, variableName: str): """Gets a configuration value """ createConfig(baseDir) configFilename=baseDir+'/config.json' - tries=0 - while tries<5: - try: - with open(configFilename, 'r') as fp: - configJson=commentjson.load(fp) - if configJson.get(variableName): - return configJson[variableName] - break - except Exception as e: - print('WARN: commentjson exception getConfigParam - '+str(e)) - time.sleep(1) - tries+=1 + configJson=loadJson(configFilename) + if configJson: + if configJson.get(variableName): + return configJson[variableName] return None diff --git a/daemon.py b/daemon.py index d680524c..14fabefd 100644 --- a/daemon.py +++ b/daemon.py @@ -123,6 +123,8 @@ from utils import getNicknameFromActor from utils import getDomainFromActor from utils import getStatusNumber from utils import urlPermitted +from utils import loadJson +from utils import saveJson from manualapprove import manualDenyFollowRequest from manualapprove import manualApproveFollowRequest from announce import createAnnounce @@ -1700,19 +1702,12 @@ class PubServer(BaseHTTPRequestHandler): self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'/outbox/'+ \ self.server.httpPrefix+':##'+self.server.domainFull+'#users#'+nickname+'#statuses#'+statusNumber+'.json' if os.path.isfile(postFilename): - postJsonObject={} + postJsonObject=loadJson(postFilename) loadedPost=False - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception 1 - '+str(e)) - time.sleep(1) - tries+=1 + if postJsonObject: + loadedPost=True + else: + postJsonObject={} if loadedPost: # Only authorized viewers get to see likes on posts # Otherwize marketers could gain more social graph info @@ -1903,19 +1898,8 @@ class PubServer(BaseHTTPRequestHandler): nickname=postSections[0] actorFilename=self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'.json' if os.path.isfile(actorFilename): - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception 2 - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: if actorJson.get('roles'): if self._requestHTTP(): getPerson = \ @@ -1970,19 +1954,8 @@ class PubServer(BaseHTTPRequestHandler): nickname=postSections[0] actorFilename=self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'.json' if os.path.isfile(actorFilename): - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception 3 - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: if actorJson.get('skills'): if self._requestHTTP(): getPerson = \ @@ -2046,20 +2019,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'/outbox/'+ \ self.server.httpPrefix+':##'+self.server.domainFull+'#users#'+nickname+'#statuses#'+statusNumber+'.json' if os.path.isfile(postFilename): - postJsonObject={} - readPost=False - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - readPost=True - break - except Exception as e: - print('WARN: commentjson exception 4 - '+str(e)) - time.sleep(1) - tries+=1 - if not readPost: + postJsonObject=loadJson(postFilename) + if not postJsonObject: self.send_response(429) self.end_headers() self.server.GETbusy=False @@ -3452,19 +3413,8 @@ class PubServer(BaseHTTPRequestHandler): actorFilename=self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'.json' if os.path.isfile(actorFilename): - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception 5 - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: actorChanged=False skillCtr=1 newSkills={} @@ -3583,29 +3533,11 @@ class PubServer(BaseHTTPRequestHandler): os.remove(allowedInstancesFilename) # save actor json file within accounts if actorChanged: - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) # also copy to the actors cache and personCache in memory storePersonInCache(self.server.baseDir,actorJson['id'],actorJson,self.server.personCache) actorCacheFilename=self.server.baseDir+'/cache/actors/'+actorJson['id'].replace('/','#')+'.json' - tries=0 - while tries<5: - try: - with open(actorCacheFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorCacheFilename) # send actor update to followers updateActorJson={ 'type': 'Update', @@ -4692,19 +4624,7 @@ def runDaemon(projectVersion, \ systemLanguage='en' translationsFile=baseDir+'/translations/'+systemLanguage+'.json' print('System language: '+systemLanguage) - - tries=0 - while tries<5: - try: - with open(translationsFile, 'r') as fp: - httpd.translate=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception 7 - '+str(e)) - print('ERROR while loading translations '+translationsFile) - print(e) - time.sleep(1) - tries+=1 + httpd.translate=loadJson(translationsFile) httpd.outboxThread={} httpd.newPostThread={} diff --git a/follow.py b/follow.py index 448fae2f..567187ee 100644 --- a/follow.py +++ b/follow.py @@ -20,6 +20,8 @@ from utils import getStatusNumber from utils import followPerson from posts import sendSignedJson from posts import getPersonBox +from utils import loadJson +from utils import saveJson from acceptreject import createAccept from acceptreject import createReject from webfinger import webfingerHandle @@ -337,17 +339,7 @@ def followApprovalRequired(baseDir: str,nicknameToFollow: str, \ domainToFollow=domainToFollow.split(':')[0] actorFilename=baseDir+'/accounts/'+nicknameToFollow+'@'+domainToFollow+'.json' if os.path.isfile(actorFilename): - actor=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actor=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception followApprovalRequired - '+str(e)) - time.sleep(1) - tries+=1 + actor=loadJson(actorFilename) if actor: if actor.get('manuallyApprovesFollowers'): manuallyApproveFollows=actor['manuallyApprovesFollowers'] @@ -412,17 +404,7 @@ def storeFollowRequest(baseDir: str, \ if not os.path.isdir(requestsDir): os.mkdir(requestsDir) followActivityfilename=requestsDir+'/'+approveHandle+'.follow' - tries=0 - while tries<5: - try: - with open(followActivityfilename, 'w') as fp: - commentjson.dump(followJson, fp, indent=2, sort_keys=False) - return True - except Exception as e: - print(e) - time.sleep(1) - tries+=1 - return False + return saveJson(followJson,followActivityfilename) def receiveFollowRequest(session,baseDir: str,httpPrefix: str, \ port: int,sendThreads: [],postLog: [], \ @@ -889,17 +871,7 @@ def getFollowersOfActor(baseDir :str,actor :str,debug: bool) -> {}: if debug: print('DEBUG: checking capabilities of'+account) if os.path.isfile(ocapFilename): - ocapJson=None - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - ocapJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getFollowersOfActor - '+str(e)) - time.sleep(1) - tries+=1 + ocapJson=loadJson(ocapFilename) if ocapJson: if ocapJson.get('id'): if debug: diff --git a/inbox.py b/inbox.py index 1a367273..8eee1412 100644 --- a/inbox.py +++ b/inbox.py @@ -23,6 +23,8 @@ from utils import locatePost from utils import deletePost from utils import removeAttachment from utils import removeModerationPostFromIndex +from utils import loadJson +from utils import saveJson from httpsig import verifyPostHeaders from session import createSession from session import getJson @@ -314,16 +316,7 @@ def savePostToInboxQueue(baseDir: str,httpPrefix: str, \ if debug: print('Inbox queue item created') pprint(newQueueItem) - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(newQueueItem, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(newQueueItem,filename) return filename def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \ @@ -348,17 +341,7 @@ def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \ queue.pop(0) return False - tries=0 - oc=None - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - oc=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception inboxCheckCapabilities - '+str(e)) - time.sleep(1) - tries+=1 + oc=loadJson(ocapFilename) if not oc: return False @@ -424,19 +407,8 @@ def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \ ocapFilename=baseDir+'/accounts/'+handle+'/ocap/accept/'+actor.replace('/','#')+'.json' if os.path.isfile(ocapFilename): # read the granted capabilities and obtain the id - loadedOcap=False - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - ocapJson=commentjson.load(fp) - loadedOcap=True - break - except Exception as e: - print('WARN: commentjson exception inboxPostRecipientsAdd - '+str(e)) - time.sleep(1) - tries+=1 - if loadedOcap: + ocapJson=loadJson(ocapFilename) + if ocapJson: if ocapJson.get('id'): # append with the capabilities id recipientsDict[handle]=ocapJson['id'] @@ -723,19 +695,8 @@ def personReceiveUpdate(baseDir: str, \ return False else: if os.path.isfile(actorFilename): - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - existingPersonJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception personReceiveUpdate - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + existingPersonJson=loadJson(actorFilename) + if existingPersonJson: if existingPersonJson['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']: if debug: print('WARN: Public key does not match cached actor when updating') @@ -743,17 +704,8 @@ def personReceiveUpdate(baseDir: str, \ # save to cache in memory storePersonInCache(baseDir,personJson['id'],personJson,personCache) # save to cache on file - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(personJson, fp, indent=2, sort_keys=False) - print('actor updated for '+personJson['id']) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + if saveJson(personJson,actorFilename): + print('actor updated for '+personJson['id']) # remove avatar if it exists so that it will be refreshed later # when a timeline is constructed @@ -1148,19 +1100,8 @@ def receiveUndoAnnounce(session,handle: str,isGroup: bool,baseDir: str, \ if debug: print('DEBUG: announced/repeated post to be undone found in inbox') - loadedPost=False - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception receiveUndoAnnounce - '+str(e)) - time.sleep(1) - tries+=1 - if loadedPost: + postJsonObject=loadJson(postFilename) + if postJsonObject: if not postJsonObject.get('type'): if postJsonObject['type']!='Announce': if debug: @@ -1348,17 +1289,7 @@ def groupHandle(baseDir: str,handle: str) -> bool: actorFile=baseDir+'/accounts/'+handle+'.json' if not os.path.isfile(actorFile): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFile, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception groupHandle - '+str(e)) - time.sleep(1) - tries+=1 + actorJson=loadJson(actorFile) if not actorJson: return False return actorJson['type']=='Group' @@ -1369,17 +1300,7 @@ def getGroupName(baseDir: str,handle: str) -> str: actorFile=baseDir+'/accounts/'+handle+'.json' if not os.path.isfile(actorFile): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFile, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getGroupName - '+str(e)) - time.sleep(1) - tries+=1 + actorJson=loadJson(actorFile) if not actorJson: return 'Group' return actorJson['name'] @@ -1659,20 +1580,7 @@ def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \ obtainAvatarForReplyPost(session,baseDir,httpPrefix,domain,personCache,postJsonObject,debug) # save the post to file - postSavedToFile=False - tries=0 - while tries<5: - try: - with open(destinationFilename, 'w+') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - postSavedToFile=True - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 - - if postSavedToFile: + if saveJson(postJsonObject,destinationFilename): if not inboxUpdateIndex(baseDir,handle,destinationFilename,debug): print('ERROR: unable to update inbox index') @@ -2062,17 +1970,7 @@ def runInboxQueue(projectVersion: str, \ if len(recipientsDictFollowers)>0: sharedInboxPostFilename=queueJson['destination'].replace(inboxHandle,inboxHandle) if not os.path.isfile(sharedInboxPostFilename): - tries=0 - while tries<5: - try: - with open(sharedInboxPostFilename, 'w') as fp: - commentjson.dump(queueJson['post'],fp,indent=2, \ - sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(queueJson['post'],sharedInboxPostFilename) # for posts addressed to specific accounts for handle,capsId in recipientsDict.items(): diff --git a/like.py b/like.py index dea55a61..74e602c5 100644 --- a/like.py +++ b/like.py @@ -16,6 +16,8 @@ from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost from utils import getCachedPostFilename +from utils import loadJson +from utils import saveJson from posts import sendSignedJson from session import postJson from webfinger import webfingerHandle @@ -25,18 +27,7 @@ from posts import getPersonBox def undoLikesCollectionEntry(baseDir: str,postFilename: str,objectUrl: str,actor: str,domain: str,debug: bool) -> None: """Undoes a like for a particular actor """ - postJsonObject=None - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception undoLikesCollectionEntry - '+str(e)) - time.sleep(1) - tries+=1 - + postJsonObject=loadJson(postFilename) if postJsonObject: # remove any cached version of this post so that the like icon is changed nickname=getNicknameFromActor(actor) @@ -80,16 +71,7 @@ def undoLikesCollectionEntry(baseDir: str,postFilename: str,objectUrl: str,actor del postJsonObject['object']['likes'] else: postJsonObject['object']['likes']['totalItems']=len(postJsonObject['likes']['items']) - tries=0 - while tries<5: - try: - with open(postFilename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(postJsonObject,postFilename) def likedByPerson(postJsonObject: {}, nickname: str,domain: str) -> bool: """Returns True if the given post is liked by the given person @@ -121,18 +103,7 @@ def noOfLikes(postJsonObject: {}) -> int: def updateLikesCollection(baseDir: str,postFilename: str,objectUrl: str, actor: str,domain: str,debug: bool) -> None: """Updates the likes collection within a post """ - postJsonObject=None - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception updateLikesCollection - '+str(e)) - time.sleep(1) - tries+=1 - + postJsonObject=loadJson(postFilename) if postJsonObject: # remove any cached version of this post so that the like icon is changed nickname=getNicknameFromActor(actor) @@ -178,16 +149,7 @@ def updateLikesCollection(baseDir: str,postFilename: str,objectUrl: str, actor: if debug: print('DEBUG: saving post with likes added') pprint(postJsonObject) - tries=0 - while tries<5: - try: - with open(postFilename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(postJsonObject,postFilename) def like(session,baseDir: str,federationList: [],nickname: str,domain: str,port: int, \ ccList: [],httpPrefix: str,objectUrl: str,clientToServer: bool, \ diff --git a/manualapprove.py b/manualapprove.py index c161ed48..055d0901 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -13,6 +13,8 @@ import time from follow import followedAccountAccepts from follow import followedAccountRejects from follow import removeFromFollowRequests +from utils import loadJson +from utils import saveJson def manualDenyFollowRequest(session,baseDir: str, \ httpPrefix: str, @@ -97,17 +99,7 @@ def manualApproveFollowRequest(session,baseDir: str, \ requestsDir=accountsDir+'/requests' followActivityfilename=requestsDir+'/'+handle+'.follow' if os.path.isfile(followActivityfilename): - followJson=None - tries=0 - while tries<5: - try: - with open(followActivityfilename, 'r') as fp: - followJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception manualApproveFollowRequest - '+str(e)) - time.sleep(1) - tries+=1 + followJson=loadJson(followActivityfilename) if followJson: approveNickname=approveHandle.split('@')[0] approveDomain=approveHandle.split('@')[1].replace('\n','') diff --git a/person.py b/person.py index c2a7ad1d..8a2868d8 100644 --- a/person.py +++ b/person.py @@ -31,6 +31,8 @@ from roles import setRole from media import removeMetaData from utils import validNickname from utils import noOfAccounts +from utils import loadJson +from utils import saveJson from auth import createPassword from config import setConfigParam from config import getConfigParam @@ -91,31 +93,11 @@ def setProfileImage(baseDir: str,httpPrefix :str,nickname: str,domain: str, \ iconFilename=iconFilenameBase+'.gif' profileFilename=baseDir+'/accounts/'+handle+'/'+iconFilename - personJson=None - tries=0 - while tries<5: - try: - with open(personFilename, 'r') as fp: - personJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setProfileImage - '+str(e)) - time.sleep(1) - tries+=1 - + personJson=loadJson(personFilename) if personJson: personJson[iconFilenameBase]['mediaType']=mediaType personJson[iconFilenameBase]['url']=httpPrefix+'://'+fullDomain+'/users/'+nickname+'/'+iconFilename - tries=0 - while tries<5: - try: - with open(personFilename, 'w') as fp: - commentjson.dump(personJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(personJson,personFilename) cmd = '/usr/bin/convert '+imageFilename+' -size '+resolution+' -quality 50 '+profileFilename subprocess.call(cmd, shell=True) @@ -136,30 +118,10 @@ def setOrganizationScheme(baseDir: str,nickname: str,domain: str, \ if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setOrganizationScheme - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: actorJson['orgSchema']=schema - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) return True def accountExists(baseDir: str,nickname: str,domain: str) -> bool: @@ -279,16 +241,7 @@ def createPersonBase(baseDir: str,nickname: str,domain: str,port: int, \ if not os.path.isdir(baseDir+peopleSubdir+'/'+handle+'/queue'): os.mkdir(baseDir+peopleSubdir+'/'+handle+'/queue') filename=baseDir+peopleSubdir+'/'+handle+'.json' - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(newPerson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(newPerson,filename) # save to cache if not os.path.isdir(baseDir+'/cache'): @@ -296,16 +249,7 @@ def createPersonBase(baseDir: str,nickname: str,domain: str,port: int, \ if not os.path.isdir(baseDir+'/cache/actors'): os.mkdir(baseDir+'/cache/actors') cacheFilename=baseDir+'/cache/actors/'+newPerson['id'].replace('/','#')+'.json' - tries=0 - while tries<5: - try: - with open(cacheFilename, 'w') as fp: - commentjson.dump(newPerson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(newPerson,cacheFilename) # save the private key privateKeysSubdir='/keys/private' @@ -453,20 +397,9 @@ def personLookup(domain: str,path: str,baseDir: str) -> {}: filename=baseDir+'/accounts/'+handle+'.json' if not os.path.isfile(filename): return None - personJson={"user": "unknown"} - tries=0 - while tries<5: - try: - with open(filename, 'r') as fp: - personJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception personLookup - '+str(e)) - print('WARN: Failed to load actor '+filename) - time.sleep(1) - tries+=1 - if tries>=5: - return None + personJson=loadJson(filename) + #if not personJson: + # personJson={"user": "unknown"} return personJson def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \ @@ -578,31 +511,11 @@ def setDisplayNickname(baseDir: str,nickname: str, domain: str, \ if not os.path.isfile(filename): return False - personJson=None - tries=0 - while tries<5: - try: - with open(filename, 'r') as fp: - personJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setDisplayNickname - '+str(e)) - time.sleep(1) - tries+=1 - + personJson=loadJson(filename) if not personJson: return False personJson['name']=displayName - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(personJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(personJson,filename) return True def setBio(baseDir: str,nickname: str, domain: str, bio: str) -> bool: @@ -613,35 +526,14 @@ def setBio(baseDir: str,nickname: str, domain: str, bio: str) -> bool: if not os.path.isfile(filename): return False - personJson=None - tries=0 - while tries<5: - try: - with open(filename, 'r') as fp: - personJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setBio - '+str(e)) - time.sleep(1) - tries+=1 - + personJson=loadJson(filename) if not personJson: return False if not personJson.get('summary'): return False personJson['summary']=bio - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(personJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 - + saveJson(personJson,filename) return True def isSuspended(baseDir: str,nickname: str) -> bool: diff --git a/posts.py b/posts.py index 37374056..ffeb868a 100644 --- a/posts.py +++ b/posts.py @@ -39,6 +39,8 @@ from utils import getDomainFromActor from utils import deletePost from utils import validNickname from utils import locatePost +from utils import loadJson +from utils import saveJson from capabilities import getOcapFilename from capabilities import capabilitiesUpdate from media import attachMedia @@ -434,16 +436,7 @@ def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \ boxDir = createPersonDir(nickname,domain,baseDir,boxname) filename=boxDir+'/'+postId.replace('/','#')+'.json' - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(postJsonObject,filename) return filename def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None: @@ -540,17 +533,7 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \ # the same warning replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo) if replyPostFilename: - replyToJson=None - tries=0 - while tries<5: - try: - with open(replyPostFilename, 'r') as fp: - replyToJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception createPostBase - '+str(e)) - time.sleep(1) - tries+=1 + replyToJson=loadJson(replyPostFilename) if replyToJson: if replyToJson.get('object'): if replyToJson['object'].get('sensitive'): @@ -605,18 +588,10 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \ ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted') if ocapFilename: if os.path.isfile(ocapFilename): - tries=0 - while tries<5: - try: - with open(ocapFilename, 'r') as fp: - oc=commentjson.load(fp) - if oc.get('id'): - capabilityIdList=[oc['id']] - break - except Exception as e: - print('WARN: commentjson exception createPostBase - '+str(e)) - time.sleep(1) - tries+=1 + oc=loadJson(ocapFilename) + if oc: + if oc.get('id'): + capabilityIdList=[oc['id']] newPost = { "@context": postContext, 'id': newPostId+'/activity', @@ -1816,17 +1791,9 @@ def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix for postUrl in pageLines: postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json' if os.path.isfile(postFilename): - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - boxItems['orderedItems'].append(postJsonObject) - break - except Exception as e: - print('WARN: commentjson exception createModeration - '+str(e)) - time.sleep(1) - tries+=1 + postJsonObject=loadJson(postFilename) + if postJsonObject: + boxItems['orderedItems'].append(postJsonObject) if headerOnly: return boxHeader @@ -2402,19 +2369,8 @@ def populateRepliesJson(baseDir: str,nickname: str,domain: str, \ if os.path.isfile(searchFilename): if authorized or \ 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read(): - loadedPost=False - tries=0 - while tries<5: - try: - with open(searchFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception populateRepliesJson - '+str(e)) - time.sleep(1) - tries+=1 - if loadedPost: + postJsonObject=loadJson(searchFilename) + if postJsonObject: if postJsonObject['object'].get('cc'): if authorized or \ ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \ @@ -2438,19 +2394,8 @@ def populateRepliesJson(baseDir: str,nickname: str,domain: str, \ if authorized or \ 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read(): # get the json of the reply and append it to the collection - loadedPost=False - tries=0 - while tries<5: - try: - with open(searchFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception populateRepliesJson 2 - '+str(e)) - time.sleep(1) - tries+=1 - if loadedPost: + postJsonObject=loadJson(searchFilename) + if postJsonObject: if postJsonObject['object'].get('cc'): if authorized or \ ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \ @@ -2489,16 +2434,9 @@ def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain: if os.path.isfile(announceFilename): print('Reading cached Announce content for '+postJsonObject['object']) - tries=0 - while tries<5: - try: - with open(announceFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - return postJsonObject - except Exception as e: - print('WARN: commentjson exception downloadAnnounce - '+str(e)) - time.sleep(1) - tries+=1 + postJsonObject=loadJson(announceFilename) + if postJsonObject: + return postJsonObject else: print('Downloading Announce content for '+postJsonObject['object']) asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'} @@ -2556,14 +2494,6 @@ def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain: rejectAnnounce(announceFilename) return None postJsonObject=announcedJson - tries=0 - while tries<5: - try: - with open(announceFilename, 'w') as fp: - commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False) - return postJsonObject - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + if saveJson(postJsonObject,announceFilename): + return postJsonObject return None diff --git a/roles.py b/roles.py index 448b80e9..3886b547 100644 --- a/roles.py +++ b/roles.py @@ -16,6 +16,8 @@ from posts import getPersonBox from session import postJson from utils import getNicknameFromActor from utils import getDomainFromActor +from utils import loadJson +from utils import saveJson def clearModeratorStatus(baseDir: str) -> None: """Removes moderator status from all accounts @@ -29,32 +31,12 @@ def clearModeratorStatus(baseDir: str) -> None: if filename.endswith(".json") and '@' in filename: filename=os.path.join(baseDir+'/accounts/', filename) if '"moderator"' in open(filename).read(): - actorJson=None - tries=0 - while tries<5: - try: - with open(filename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception clearModeratorStatus - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(filename) if actorJson: if actorJson['roles'].get('instance'): if 'moderator' in actorJson['roles']['instance']: actorJson['roles']['instance'].remove('moderator') - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,filename) def addModerator(baseDir: str,nickname: str,domain: str) -> None: """Adds a moderator nickname to the file @@ -108,18 +90,7 @@ def setRole(baseDir: str,nickname: str,domain: str, \ if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setRole - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: if role: # add the role @@ -139,16 +110,7 @@ def setRole(baseDir: str,nickname: str,domain: str, \ # if the project contains no roles then remove it if len(actorJson['roles'][project])==0: del actorJson['roles'][project] - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) return True def getRoles(baseDir: str,nickname: str,domain: str, \ @@ -159,18 +121,7 @@ def getRoles(baseDir: str,nickname: str,domain: str, \ if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getRoles - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: if not actorJson.get('roles'): return None diff --git a/shares.py b/shares.py index 718511b2..8a47e1eb 100644 --- a/shares.py +++ b/shares.py @@ -18,6 +18,8 @@ from session import postJson from utils import validNickname from utils import getNicknameFromActor from utils import getDomainFromActor +from utils import loadJson +from utils import saveJson from media import removeMetaData def removeShare(baseDir: str,nickname: str,domain: str, \ @@ -26,16 +28,7 @@ def removeShare(baseDir: str,nickname: str,domain: str, \ """ sharesFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/shares.json' if os.path.isfile(sharesFilename): - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception removeShare - '+str(e)) - time.sleep(1) - tries+=1 + sharesJson=loadJson(sharesFilename) itemID=displayName.replace(' ','') if sharesJson.get(itemID): @@ -50,16 +43,7 @@ def removeShare(baseDir: str,nickname: str,domain: str, \ os.remove(itemIDfile+'.gif') # remove the item itself del sharesJson[itemID] - tries=0 - while tries<5: - try: - with open(sharesFilename, 'w') as fp: - commentjson.dump(sharesJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(sharesJson,sharesFilename) def addShare(baseDir: str, \ httpPrefix: str,nickname: str,domain: str,port: int, \ @@ -76,16 +60,7 @@ def addShare(baseDir: str, \ sharesFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/shares.json' sharesJson={} if os.path.isfile(sharesFilename): - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception addShare - '+str(e)) - time.sleep(1) - tries+=1 + sharesJson=loadJson(sharesFilename) duration=duration.lower() durationSec=0 @@ -161,16 +136,7 @@ def addShare(baseDir: str, \ "expire": durationSec } - tries=0 - while tries<5: - try: - with open(sharesFilename, 'w') as fp: - commentjson.dump(sharesJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(sharesJson,sharesFilename) def expireShares(baseDir: str) -> None: """Removes expired items from shares @@ -192,17 +158,7 @@ def expireSharesForAccount(baseDir: str,nickname: str,domain: str) -> None: handle=nickname+'@'+handleDomain sharesFilename=baseDir+'/accounts/'+handle+'/shares.json' if os.path.isfile(sharesFilename): - sharesJson=None - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception expireSharesForAccount - '+str(e)) - time.sleep(1) - tries+=1 + sharesJson=loadJson(sharesFilename) if sharesJson: currTime=int(time.time()) deleteItemID=[] @@ -220,16 +176,7 @@ def expireSharesForAccount(baseDir: str,nickname: str,domain: str) -> None: os.remove(itemIDfile+'.jpg') if os.path.isfile(itemIDfile+'.gif'): os.remove(itemIDfile+'.gif') - tries=0 - while tries<5: - try: - with open(sharesFilename, 'w') as fp: - commentjson.dump(sharesJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print('WARN: commentjson exception expireSharesForAccount 2 - '+str(e)) - time.sleep(1) - tries+=1 + saveJson(sharesJson,sharesFilename) def getSharesFeedForPerson(baseDir: str, \ domain: str,port: int, \ @@ -280,17 +227,9 @@ def getSharesFeedForPerson(baseDir: str, \ if headerOnly: noOfShares=0 if os.path.isfile(sharesFilename): - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - noOfShares=len(sharesJson.items()) - break - except Exception as e: - print('WARN: commentjson exception getSharesFeedForPerson - '+str(e)) - time.sleep(1) - tries+=1 + sharesJson=loadJson(sharesFilename) + if sharesJson: + noOfShares=len(sharesJson.items()) shares = { '@context': 'https://www.w3.org/ns/activitystreams', 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/shares?page=1', @@ -318,18 +257,7 @@ def getSharesFeedForPerson(baseDir: str, \ pageCtr=0 totalCtr=0 - sharesJson=None - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getSharesFeedForPerson 2 - '+str(e)) - time.sleep(1) - tries+=1 - + sharesJson=loadJson(sharesFilename) if sharesJson: for itemID,item in sharesJson.items(): pageCtr += 1 diff --git a/skills.py b/skills.py index a9bef6e1..02689b9f 100644 --- a/skills.py +++ b/skills.py @@ -16,6 +16,8 @@ from posts import getPersonBox from session import postJson from utils import getNicknameFromActor from utils import getDomainFromActor +from utils import loadJson +from utils import saveJson def setSkillLevel(baseDir: str,nickname: str,domain: str, \ skill: str,skillLevelPercent: int) -> bool: @@ -28,18 +30,7 @@ def setSkillLevel(baseDir: str,nickname: str,domain: str, \ if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setSkillLevel - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: if not actorJson.get('skills'): actorJson['skills']={} @@ -47,16 +38,7 @@ def setSkillLevel(baseDir: str,nickname: str,domain: str, \ actorJson['skills'][skill]=skillLevelPercent else: del actorJson['skills'][skill] - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) return True def setSkills(baseDir: str,nickname: str,domain: str,skills: {}) -> None: @@ -64,30 +46,10 @@ def setSkills(baseDir: str,nickname: str,domain: str,skills: {}) -> None: if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception setSkills - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: actorJson['skills']=skills - tries=0 - while tries<5: - try: - with open(actorFilename, 'w') as fp: - commentjson.dump(actorJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(actorJson,actorFilename) def getSkills(baseDir: str,nickname: str,domain: str) -> []: """Returns the skills for a given person @@ -96,18 +58,7 @@ def getSkills(baseDir: str,nickname: str,domain: str) -> []: if not os.path.isfile(actorFilename): return False - actorJson=None - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getSkills - '+str(e)) - time.sleep(1) - tries+=1 - + actorJson=loadJson(actorFilename) if actorJson: if not actorJson.get('skills'): return None diff --git a/utils.py b/utils.py index 3d225d25..8494eac4 100644 --- a/utils.py +++ b/utils.py @@ -12,6 +12,37 @@ import shutil import datetime import commentjson +def saveJson(jsonObject: {},filename: str) -> bool: + """Saves json to a file + """ + tries=0 + while tries<5: + try: + with open(filename, 'w') as fp: + commentjson.dump(jsonObject, fp, indent=2, sort_keys=False) + return True + except Exception as e: + print(e) + time.sleep(1) + tries+=1 + return False + +def loadJson(filename: str) -> {}: + """Makes a few attempts to load a json formatted file + """ + jsonObject=None + tries=0 + while tries<5: + try: + with open(filename, 'r') as fp: + jsonObject=commentjson.load(fp) + break + except Exception as e: + print('WARN: loadJson exception - '+str(e)) + time.sleep(2) + tries+=1 + return jsonObject + def getStatusNumber() -> (str,str): """Returns the status number and published date """ diff --git a/webfinger.py b/webfinger.py index 28af3f99..72521c84 100644 --- a/webfinger.py +++ b/webfinger.py @@ -17,6 +17,8 @@ import time from session import getJson from cache import storeWebfingerInCache from cache import getWebfingerFromCache +from utils import loadJson +from utils import saveJson def parseHandle(handle: str) -> (str,str): if '.' not in handle: @@ -92,29 +94,11 @@ def storeWebfingerEndpoint(nickname: str,domain: str,port: int,baseDir: str, \ if not os.path.isdir(baseDir+wfSubdir): os.mkdir(baseDir+wfSubdir) filename=baseDir+wfSubdir+'/'+handle.lower()+'.json' - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(wfJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(wfJson,filename) if nickname=='inbox': handle=originalDomain+'@'+domain filename=baseDir+wfSubdir+'/'+handle.lower()+'.json' - tries=0 - while tries<5: - try: - with open(filename, 'w') as fp: - commentjson.dump(wfJson, fp, indent=2, sort_keys=False) - break - except Exception as e: - print(e) - time.sleep(1) - tries+=1 + saveJson(wfJson,filename) return True def createWebfingerEndpoint(nickname: str,domain: str,port: int, \ @@ -232,15 +216,7 @@ def webfingerLookup(path: str,baseDir: str,port: int,debug: bool) -> {}: if debug: print('DEBUG: WEBFINGER filename not found '+filename) return None - wfJson={"nickname": "unknown"} - tries=0 - while tries<5: - try: - with open(filename, 'r') as fp: - wfJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception webfingerLookup - '+str(e)) - time.sleep(1) - tries+=1 + wfJson=loadJson(filename) + if not wfJson: + wfJson={"nickname": "unknown"} return wfJson diff --git a/webinterface.py b/webinterface.py index d8365731..92f99eab 100644 --- a/webinterface.py +++ b/webinterface.py @@ -25,6 +25,8 @@ from utils import isPublicPost from utils import getDisplayName from utils import getCachedPostDirectory from utils import getCachedPostFilename +from utils import loadJson +from utils import saveJson from follow import isFollowingActor from webfinger import webfingerHandle from posts import isDM @@ -154,20 +156,9 @@ def htmlSearchEmoji(translate: {},baseDir: str,searchStr: str) -> str: emojiForm+='
'+translate['No results']+'
' emojiForm+=htmlFooter() return emojiForm - - loadedEmoji=False - tries=0 - while tries<5: - try: - with open(emojiLookupFilename, 'r') as fp: - emojiJson=commentjson.load(fp) - loadedEmoji=True - break - except Exception as e: - print('WARN: commentjson exception htmlSearchEmoji - '+str(e)) - time.sleep(1) - tries+=1 - if loadedEmoji: + + emojiJson=loadJson(emojiLookupFilename) + if emojiJson: results={} for emojiName,filename in emojiJson.items(): if searchStr in emojiName: @@ -229,17 +220,7 @@ def htmlSearchSharedItems(translate: {}, \ if not os.path.isfile(sharesFilename): continue - sharesJson=None - tries=0 - while tries<5: - try: - with open(sharesFilename, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception htmlSearchSharedItems - '+str(e)) - time.sleep(1) - tries+=1 + sharesJson=loadJson(sharesFilename) if not sharesJson: continue @@ -395,19 +376,8 @@ def htmlHashtagSearch(translate: {}, \ if not postFilename: index-=1 continue - loadedPost=False - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception htmlHashtagSearch - '+str(e)) - time.sleep(1) - tries+=1 - if loadedPost: + postJsonObject=loadJson(postFilename) + if postJsonObject: if not isPublicPost(postJsonObject): index-=1 continue @@ -447,19 +417,8 @@ def htmlSkillsSearch(translate: {},baseDir: str, \ if f.startswith('inbox@'): continue actorFilename = os.path.join(subdir, f) - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception htmlSkillsSearch - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: if actorJson.get('id') and \ actorJson.get('skills') and \ actorJson.get('name') and \ @@ -487,19 +446,8 @@ def htmlSkillsSearch(translate: {},baseDir: str, \ if f.startswith('inbox@'): continue actorFilename = os.path.join(subdir, f) - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - cachedActorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception htmlSkillsSearch - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + cachedActorJson=loadJson(actorFilename) + if cachedActorJson: if cachedActorJson.get('actor'): actorJson=cachedActorJson['actor'] if actorJson.get('id') and \ @@ -573,19 +521,8 @@ def htmlEditProfile(translate: {},baseDir: str,path: str,domain: str,port: int) displayNickname=nickname bioStr='' manuallyApprovesFollowers='' - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception htmlEditProfile - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: if actorJson.get('name'): displayNickname=actorJson['name'] if actorJson.get('summary'): @@ -1616,19 +1553,8 @@ def followerApprovalActive(baseDir: str,nickname: str,domain: str) -> bool: manuallyApprovesFollowers=False actorFilename=baseDir+'/accounts/'+nickname+'@'+domain+'.json' if os.path.isfile(actorFilename): - loadedActor=False - tries=0 - while tries<5: - try: - with open(actorFilename, 'r') as fp: - actorJson=commentjson.load(fp) - loadedActor=True - break - except Exception as e: - print('WARN: commentjson exception followerApprovalActive - '+str(e)) - time.sleep(1) - tries+=1 - if loadedActor: + actorJson=loadJson(actorFilename) + if actorJson: if actorJson.get('manuallyApprovesFollowers'): manuallyApprovesFollowers=actorJson['manuallyApprovesFollowers'] return manuallyApprovesFollowers @@ -2460,19 +2386,8 @@ def htmlIndividualPost(translate: {}, \ postFilename=locatePost(baseDir,nickname,domain,postJsonObject['object']['inReplyTo']) if not postFilename: break - loadedPost=False - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - loadedPost=True - break - except Exception as e: - print('WARN: commentjson exception htmlIndividualPost - '+str(e)) - time.sleep(1) - tries+=1 - if loadedPost: + postJsonObject=loadJson(postFilename) + if postJsonObject: postStr= \ individualPostAsHtml(iconsDir,translate,None, \ baseDir,session,wfRequest,personCache, \ @@ -2537,18 +2452,7 @@ def htmlRemoveSharedItem(translate: {},baseDir: str,actor: str,shareName: str) - sharesFile=baseDir+'/accounts/'+nickname+'@'+domain+'/shares.json' if not os.path.isfile(sharesFile): return None - sharesJson=None - tries=0 - while tries<5: - try: - with open(sharesFile, 'r') as fp: - sharesJson=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception htmlRemoveSharedItem - '+str(e)) - time.sleep(1) - tries+=1 - + sharesJson=loadJson(sharesFile) if not sharesJson: return None if not sharesJson.get(shareName): @@ -2604,17 +2508,7 @@ def htmlDeletePost(translate,pageNumber: int, \ if not postFilename: return None - postJsonObject=None - tries=0 - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception htmlDeletePost - '+str(e)) - time.sleep(1) - tries+=1 + postJsonObject=loadJson(postFilename) if not postJsonObject: return None @@ -2918,18 +2812,7 @@ def getCalendarEvents(baseDir: str,nickname: str,domain: str,year: int,monthNumb postId=postId.replace('\n','') postFilename=locatePost(baseDir,nickname,domain,postId) if postFilename: - postJsonObject=None - tries=0 - postJsonObject=None - while tries<5: - try: - with open(postFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - break - except Exception as e: - print('WARN: commentjson exception getCalendarEvents - '+str(e)) - time.sleep(1) - tries+=1 + postJsonObject=loadJson(postFilename) if postJsonObject: if postJsonObject.get('object'): if isinstance(postJsonObject['object'], dict):