Add some retries

announce.py

@@ -6,6 +6,7 @@ __maintainer__ = "Bob Mottram"
 __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
+import time
 import json
 import commentjson
 from pprint import pprint
@@ -68,12 +69,16 @@ def undoAnnounceCollectionEntry(postFilename: str,actor: str,debug: bool) -> Non
     to shared items in shares.py. It's shares of posts, not shares of physical objects.
     """
     postJsonObject=None
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
-
+            time.sleep(2)
+            tries+=1
     if postJsonObject:
         if not postJsonObject.get('type'):
             return
@@ -121,11 +126,16 @@ def updateAnnounceCollection(postFilename: str,actor: str,debug: bool) -> None:
     It's shares of posts, not shares of physical objects.
     """
     postJsonObject=None
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(2)
+            tries+=1
     if postJsonObject:
         if not postJsonObject.get('object'):
             if debug:

availability.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
+import time
 import commentjson
 import os
 from webfinger import webfingerHandle
@@ -27,11 +28,16 @@ def setAvailability(baseDir: str,nickname: str,domain: str, \
     if not os.path.isfile(actorFilename):
         return False
     actorJson=None
+    tries=0
+    while tries<5:
         try:
             with open(actorFilename, 'r') as fp:
                 actorJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(2)
+            tries+=1
     if actorJson:
         actorJson['availability']=status
         try:
@@ -48,11 +54,16 @@ def getAvailability(baseDir: str,nickname: str,domain: str) -> str:
     if not os.path.isfile(actorFilename):
         return False
     actorJson=None
+    tries=0
+    while tries<5:
         try:
             with open(actorFilename, 'r') as fp:
                 actorJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(2)
+            tries+=1
     if actorJson:
         if not actorJson.get('availability'):
             return None

cache.py

@@ -41,12 +41,17 @@ def getPersonFromCache(baseDir: str,personUrl: str,personCache: {}) -> {}:
         cacheFilename=baseDir+'/cache/actors/'+personUrl.replace('/','#')+'.json'
         if os.path.isfile(cacheFilename):
             personJson=None
+            tries=0
+            while tries<5:
                 try:
                     with open(cacheFilename, 'r') as fp:
                         personJson=commentjson.load(fp)
+                        break
                 except Exception as e:
                     print('ERROR: unable to load actor from cache '+cacheFilename)
                     print(e)
+                    time.sleep(2)
+                    tries+=1
             if personJson:
                 storePersonInCache(baseDir,personUrl,personJson,personCache)
                 loadedFromFile=True

daemon.py

@@ -768,8 +768,16 @@ class PubServer(BaseHTTPRequestHandler):
         # Note that this comes before the busy flag to avoid conflicts
         if self.path.endswith('.css'):
             if os.path.isfile('epicyon-profile.css'):
+                tries=0
+                while tries<5:
+                    try:
                         with open('epicyon-profile.css', 'r') as cssfile:
                             css = cssfile.read()
+                            break
+                    except Exception as e:
+                        print(e)
+                        time.sleep(1)
+                        tries+=1
                 msg=css.encode('utf-8')
                 self._set_headers('text/css',len(msg),cookie)
                 self.wfile.write(msg)
@@ -2917,12 +2925,17 @@ class PubServer(BaseHTTPRequestHandler):
                 actorFilename=self.server.baseDir+'/accounts/'+nickname+'@'+self.server.domain+'.json'
                 if os.path.isfile(actorFilename):
                     loadedActor=False
+                    tries=0
+                    while tries<5:
                         try:
                             with open(actorFilename, 'r') as fp:
                                 actorJson=commentjson.load(fp)
                                 loadedActor=True
+                                break
                         except Exception as e:
                             print(e)
+                            time.sleep(1)
+                            tries+=1
                     if loadedActor:                    
                         actorChanged=False
                         skillCtr=1
@@ -4012,12 +4025,17 @@ def runDaemon(projectVersion, \
             translationsFile=baseDir+'/translations/'+systemLanguage+'.json'
         print('System language: '+systemLanguage)
 
+        tries=0
+        while tries<5:
             try:
                 with open(translationsFile, 'r') as fp:
                     httpd.translate=commentjson.load(fp)
+                    break
             except Exception as e:
                 print('ERROR while loading translations '+translationsFile)
                 print(e)
+                time.sleep(1)
+                tries+=1
 
     httpd.outboxThread={}
     httpd.newPostThread={}

follow.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
+import time
 import commentjson
 from pprint import pprint
 import os
@@ -336,11 +337,16 @@ def followApprovalRequired(baseDir: str,nicknameToFollow: str, \
     actorFilename=baseDir+'/accounts/'+nicknameToFollow+'@'+domainToFollow+'.json'
     if os.path.isfile(actorFilename):
         actor=None
+        tries=0
+        while tries<5:
             try:
                 with open(actorFilename, 'r') as fp:
                     actor=commentjson.load(fp)
+                    break
             except Exception as e:
                 print(e)
+                time.sleep(1)
+                tries+=1
         if actor:
             if actor.get('manuallyApprovesFollowers'):
                 manuallyApproveFollows=actor['manuallyApprovesFollowers']
@@ -864,11 +870,16 @@ def getFollowersOfActor(baseDir :str,actor :str,debug: bool) -> {}:
                             print('DEBUG: checking capabilities of'+account)
                         if os.path.isfile(ocapFilename):
                             ocapJson=None
+                            tries=0
+                            while tries<5:
                                 try:
                                     with open(ocapFilename, 'r') as fp:
                                         ocapJson=commentjson.load(fp)
+                                        break
                                 except Exception as e:
                                     print(e)
+                                    time.sleep(1)
+                                    tries+=1
                             if ocapJson:
                                 if ocapJson.get('id'):
                                     if debug:

inbox.py

@@ -312,11 +312,18 @@ def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \
             queue.pop(0)
             return False
 
+    tries=0
+    oc=None
+    while tries<5:
         try:
             with open(ocapFilename, 'r') as fp:
                 oc=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
+    if not oc: 
         return False
 
     if not oc.get('id'):
@@ -374,12 +381,17 @@ def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \
                 if os.path.isfile(ocapFilename):
                     # read the granted capabilities and obtain the id
                     loadedOcap=False
+                    tries=0
+                    while tries<5:
                         try:
                             with open(ocapFilename, 'r') as fp:
                                 ocapJson=commentjson.load(fp)
                                 loadedOcap=True
+                                break
                         except Exception as e:
                             print(e)
+                            time.sleep(1)
+                            tries+=1
                     if loadedOcap:
                         if ocapJson.get('id'):
                             # append with the capabilities id
@@ -660,12 +672,17 @@ def personReceiveUpdate(baseDir: str, \
     else:
         if os.path.isfile(actorFilename):
             loadedActor=False
+            tries=0
+            while tries<5:
                 try:
                     with open(actorFilename, 'r') as fp:
                         existingPersonJson=commentjson.load(fp)
                         loadedActor=True
+                        break
                 except Exception as e:
                     print(e)
+                    time.sleep(1)
+                    tries+=1
             if loadedActor:
                 if existingPersonJson['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
                     if debug:
@@ -1066,12 +1083,17 @@ def receiveUndoAnnounce(session,handle: str,isGroup: bool,baseDir: str, \
         print('DEBUG: announced/repeated post to be undone found in inbox')
 
     loadedPost=False
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
                 loadedPost=True
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
     if loadedPost:
         if not postJsonObject.get('type'):
             if postJsonObject['type']!='Announce':
@@ -1251,11 +1273,16 @@ def groupHandle(baseDir: str,handle: str) -> bool:
     if not os.path.isfile(actorFile):
         return False
     actorJson=None
+    tries=0
+    while tries<5:
         try:
             with open(actorFile, 'r') as fp:
                 actorJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
     if not actorJson:
         return False
     return actorJson['type']=='Group'
@@ -1267,11 +1294,16 @@ def getGroupName(baseDir: str,handle: str) -> str:
     if not os.path.isfile(actorFile):
         return False
     actorJson=None
+    tries=0
+    while tries<5:
         try:
             with open(actorFile, 'r') as fp:
                 actorJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
     if not actorJson:
         return 'Group'
     return actorJson['name']

like.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
+import time
 import commentjson
 from pprint import pprint
 from utils import urlPermitted
@@ -23,11 +24,16 @@ def undoLikesCollectionEntry(postFilename: str,objectUrl: str,actor: str,debug:
     """Undoes a like for a particular actor
     """
     postJsonObject=None
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if postJsonObject:
         if not postJsonObject.get('type'):
@@ -103,11 +109,16 @@ def updateLikesCollection(postFilename: str,objectUrl: str, actor: str,debug: bo
     """Updates the likes collection within a post
     """
     postJsonObject=None
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if postJsonObject:
         if not postJsonObject.get('object'):

manualapprove.py

@@ -98,11 +98,16 @@ def manualApproveFollowRequest(session,baseDir: str, \
                 followActivityfilename=requestsDir+'/'+handle+'.follow'
                 if os.path.isfile(followActivityfilename):
                     followJson=None
+                    tries=0
+                    while tries<5:
                         try:
                             with open(followActivityfilename, 'r') as fp:
                                 followJson=commentjson.load(fp)
+                                break
                         except Exception as e:
                             print(e)
+                            time.sleep(1)
+                            tries+=1
                     if followJson:
                         approveNickname=approveHandle.split('@')[0]
                         approveDomain=approveHandle.split('@')[1].replace('\n','')

person.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
+import time
 import commentjson
 import os
 import fileinput
@@ -91,11 +92,16 @@ def setProfileImage(baseDir: str,httpPrefix :str,nickname: str,domain: str, \
     profileFilename=baseDir+'/accounts/'+handle+'/'+iconFilename
 
     personJson=None
+    tries=0
+    while tries<5:
         try:
             with open(personFilename, 'r') as fp:
                 personJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if personJson:
         personJson[iconFilenameBase]['mediaType']=mediaType
@@ -126,11 +132,16 @@ def setOrganizationScheme(baseDir: str,nickname: str,domain: str, \
         return False
 
     actorJson=None
+    tries=0
+    while tries<5:
         try:
             with open(actorFilename, 'r') as fp:
                 actorJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if actorJson:
         actorJson['orgSchema']=schema
@@ -547,11 +558,16 @@ def setDisplayNickname(baseDir: str,nickname: str, domain: str, \
         return False
 
     personJson=None
+    tries=0
+    while tries<5:
         try:
             with open(filename, 'r') as fp:
                 personJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
             
     if not personJson:
         return False
@@ -572,11 +588,16 @@ def setBio(baseDir: str,nickname: str, domain: str, bio: str) -> bool:
         return False
 
     personJson=None
+    tries=0
+    while tries<5:
         try:
             with open(filename, 'r') as fp:
                 personJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if not personJson:
         return False

posts.py

@@ -517,11 +517,16 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
         replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo)
         if replyPostFilename:
             replyToJson=None
+            tries=0
+            while tries<5:
                 try:
                     with open(replyPostFilename, 'r') as fp:
                         replyToJson=commentjson.load(fp)
+                        break
                 except Exception as e:
                     print(e)
+                    time.sleep(1)
+                    tries+=1
             if replyToJson:
                 if replyToJson.get('object'):
                     if replyToJson['object'].get('sensitive'):
@@ -566,13 +571,18 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
         ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
         if ocapFilename:
             if os.path.isfile(ocapFilename):
+                tries=0
+                while tries<5:
                     try:
                         with open(ocapFilename, 'r') as fp:
                             oc=commentjson.load(fp)
                             if oc.get('id'):
                                 capabilityIdList=[oc['id']]
+                            break
                     except Exception as e:
                         print(e)
+                        time.sleep(1)
+                        tries+=1
         newPost = {
             "@context": "https://www.w3.org/ns/activitystreams",
             'id': newPostId+'/activity',
@@ -1719,12 +1729,17 @@ def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix
             for postUrl in pageLines:
                 postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
                 if os.path.isfile(postFilename):
+                    tries=0
+                    while tries<5:
                         try:
                             with open(postFilename, 'r') as fp:
                                 postJsonObject=commentjson.load(fp)
                                 boxItems['orderedItems'].append(postJsonObject)
+                                break
                         except Exception as e:
                             print(e)
+                            time.sleep(1)
+                            tries+=1
 
     if headerOnly:
         return boxHeader
@@ -1876,12 +1891,17 @@ def createBoxBase(session,baseDir: str,boxname: str, \
                 sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
                 # get the actor from the shared post
                 loadedPost=False
+                tries=0
+                while tries<5:
                     try:
                         with open(sharedInboxFilename, 'r') as fp:
                             postJsonObject=commentjson.load(fp)
                             loadedPost=True
+                            break
                     except Exception as e:
                         print(e)
+                        time.sleep(1)
+                        tries+=1
                 if loadedPost:                
                     actorNickname=getNicknameFromActor(postJsonObject['actor'])
                     actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
@@ -1899,12 +1919,17 @@ def createBoxBase(session,baseDir: str,boxname: str, \
                                 if os.path.isfile(ocapFilename):
                                     # read the capabilities id
                                     loadedOcap=False
+                                    tries=0
+                                    while tries<5:
                                         try:
                                             with open(ocapFilename, 'r') as fp:
                                                 ocapJson=commentjson.load(fp)
                                                 loadedOcap=True
+                                                break
                                         except Exception as e:
                                             print(e)
+                                            time.sleep(1)
+                                            tries+=1
                                     if loadedOcap:
                                         if ocapJson.get('id'):
                                             if ocapJson['id'] in capsList:                                    
@@ -2214,12 +2239,17 @@ def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
                     if authorized or \
                        'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
                         loadedPost=False
+                        tries=0
+                        while tries<5:
                             try:
                                 with open(searchFilename, 'r') as fp:
                                     postJsonObject=commentjson.load(fp)
                                     loadedPost=True
+                                    break
                             except Exception as e:
                                 print(e)
+                                time.sleep(1)
+                                tries+=1
                         if loadedPost:
                             if postJsonObject['object'].get('cc'):                                                            
                                 if authorized or \
@@ -2245,12 +2275,17 @@ def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
                        'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
                         # get the json of the reply and append it to the collection
                         loadedPost=False
+                        tries=0
+                        while tries<5:
                             try:
                                 with open(searchFilename, 'r') as fp:
                                     postJsonObject=commentjson.load(fp)
                                     loadedPost=True
+                                    break
                             except Exception as e:
                                 print(e)
+                                time.sleep(1)
+                                tries+=1
                         if loadedPost:
                             if postJsonObject['object'].get('cc'):                                                            
                                 if authorized or \
@@ -2290,12 +2325,16 @@ def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain:
 
     if os.path.isfile(announceFilename):
         print('Reading cached Announce content for '+postJsonObject['object'])
+        tries=0
+        while tries<5:
             try:
                 with open(announceFilename, 'r') as fp:
                     postJsonObject=commentjson.load(fp)
                     return postJsonObject
             except Exception as e:
                 print(e)
+                time.sleep(1)
+                tries+=1
     else:
         print('Downloading Announce content for '+postJsonObject['object'])
         asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}

utils.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import os
+import time
 import shutil
 import datetime
 import commentjson
@@ -84,11 +85,16 @@ def getDisplayName(baseDir: str,actor: str,personCache: {}) -> str:
         cachedActorFilename=baseDir+'/cache/actors/'+actor.replace('/','#')+'.json'
         if os.path.isfile(cachedActorFilename):
             actorJson=None
+            tries=0
+            while tries<5:
                 try:
                     with open(cachedActorFilename, 'r') as fp:
                         actorJson=commentjson.load(fp)
+                        break
                 except Exception as e:
                     print(e)
+                    time.sleep(1)
+                    tries+=1
             if actorJson:
                 if actorJson.get('name'):
                     return(actorJson['name'])
@@ -243,11 +249,16 @@ def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilena
     """Recursively deletes a post and its replies and attachments
     """
     postJsonObject=None
+    tries=0
+    while tries<5:
         try:
             with open(postFilename, 'r') as fp:
                 postJsonObject=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
 
     if postJsonObject:
         # remove any attachment

webfinger.py

@@ -13,6 +13,7 @@ import requests
 import json
 import commentjson
 import os
+import time
 from session import getJson
 from cache import storeWebfingerInCache
 from cache import getWebfingerFromCache
@@ -222,9 +223,14 @@ def webfingerLookup(path: str,baseDir: str,port: int,debug: bool) -> {}:
             print('DEBUG: WEBFINGER filename not found '+filename)
         return None
     wfJson={"nickname": "unknown"}
+    tries=0
+    while tries<5:
         try:
             with open(filename, 'r') as fp:
                 wfJson=commentjson.load(fp)
+                break
         except Exception as e:
             print(e)
+            time.sleep(1)
+            tries+=1
     return wfJson

webinterface.py

@@ -2766,11 +2766,17 @@ def getCalendarEvents(baseDir: str,nickname: str,domain: str,year: int,monthNumb
             postFilename=locatePost(baseDir,nickname,domain,postId)
             if postFilename:
                 postJsonObject=None
+                tries=0
+                postJsonObject=None
+                while tries<5:
                     try:
                         with open(postFilename, 'r') as fp:
                             postJsonObject=commentjson.load(fp)
+                            break
                     except Exception as e:
                         print(e)
+                        time.sleep(2)
+                        tries+=1
                 if postJsonObject:
                     if postJsonObject.get('object'):
                         if isinstance(postJsonObject['object'], dict):