diff --git a/acceptreject.py b/acceptreject.py index 7bac52a0..20908a37 100644 --- a/acceptreject.py +++ b/acceptreject.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import json -import commentjson from capabilities import capabilitiesAccept from capabilities import capabilitiesGrantedSave from utils import getStatusNumber diff --git a/announce.py b/announce.py index 5f710603..02e3f4ac 100644 --- a/announce.py +++ b/announce.py @@ -9,7 +9,6 @@ __status__ = "Production" import os import time import json -import commentjson from pprint import pprint from utils import getStatusNumber from utils import createOutboxDir diff --git a/availability.py b/availability.py index 38920bc2..d52551fb 100644 --- a/availability.py +++ b/availability.py @@ -8,7 +8,6 @@ __status__ = "Production" import json import time -import commentjson import os from webfinger import webfingerHandle from auth import createBasicAuthHeader diff --git a/bookmarks.py b/bookmarks.py index 0b311ae1..44b39c48 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -9,7 +9,6 @@ __status__ = "Production" import os import json import time -import commentjson from pprint import pprint from utils import urlPermitted from utils import getNicknameFromActor diff --git a/cache.py b/cache.py index 12c933ba..7fa562e3 100644 --- a/cache.py +++ b/cache.py @@ -9,7 +9,6 @@ __status__ = "Production" import os import time import datetime -import commentjson from utils import loadJson from utils import saveJson diff --git a/capabilities.py b/capabilities.py index e1845351..3879fa49 100644 --- a/capabilities.py +++ b/capabilities.py @@ -10,7 +10,6 @@ import os import datetime import time import json -import commentjson from auth import createPassword from utils import getNicknameFromActor from utils import getDomainFromActor diff --git a/config.py b/config.py index 7cbf7644..93e43858 100644 --- a/config.py +++ b/config.py @@ -9,7 +9,6 @@ __status__ = "Production" import os import time import json -import commentjson from utils import loadJson from utils import saveJson diff --git a/content.py b/content.py index 36c633d0..1cc3912c 100644 --- a/content.py +++ b/content.py @@ -8,9 +8,9 @@ __status__ = "Production" import os import time -import commentjson import email.parser from shutil import copyfile +from utils import loadJson def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str: """Uses the tags to replace :emoji: with html image markup @@ -341,21 +341,7 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \ # will be retained even if default_emoji.json is subsequently updated if not os.path.isfile(baseDir+'/emoji/emoji.json'): copyfile(baseDir+'/emoji/default_emoji.json',baseDir+'/emoji/emoji.json') - emojiDictCtr=0 - while not emojiDict and emojiDictCtr<5: - if emojiDictCtr>0: - print('Retry emoji load '+baseDir+'/emoji/emoji.json') - try: - with open(baseDir+'/emoji/emoji.json', 'r') as fp: - emojiDict=commentjson.load(fp) - if emojiDictCtr>0: - print('emojiDict loaded on try '+str(emojiDictCtr)) - break - except: - print('WARN: commentjson exception addHtmlTags') - print('Failed to load emoji (try '+str(emojiDictCtr)+'): '+baseDir+'/emoji/emoji.json') - time.sleep(1) - emojiDictCtr+=1 + emojiDict=loadJson(baseDir+'/emoji/emoji.json') #print('TAG: looking up emoji for :'+wordStr2+':') addEmoji(baseDir,':'+wordStr2+':',httpPrefix,originalDomain,replaceEmoji,hashtags,emojiDict) diff --git a/daemon.py b/daemon.py index 33d0a036..842527b4 100644 --- a/daemon.py +++ b/daemon.py @@ -8,7 +8,6 @@ __status__ = "Production" from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer #import socketserver -import commentjson import json import time import base64 @@ -817,12 +816,6 @@ class PubServer(BaseHTTPRequestHandler): timeDiff=int((time.time()-beginSaveTime)*1000) if timeDiff>200: print('SLOW: slow save of inbox queue item '+queueFilename+' took '+str(timeDiff)+' mS') - try: - with open(queueFilename, 'r') as fp: - queueJson=commentjson.load(fp) - print('SLOW: '+str(queueJson)) - except: - pass self.send_response(201) self.end_headers() self.server.POSTbusy=False diff --git a/delete.py b/delete.py index 0f737998..73a8bf20 100644 --- a/delete.py +++ b/delete.py @@ -8,7 +8,6 @@ __status__ = "Production" import os import json -import commentjson from utils import getStatusNumber from utils import createOutboxDir from utils import urlPermitted diff --git a/follow.py b/follow.py index 428b71b2..179d34fa 100644 --- a/follow.py +++ b/follow.py @@ -8,7 +8,6 @@ __status__ = "Production" import json import time -import commentjson from pprint import pprint import os import sys diff --git a/inbox.py b/inbox.py index 243ee9cb..e9429339 100644 --- a/inbox.py +++ b/inbox.py @@ -11,7 +11,6 @@ import os import datetime import time import json -import commentjson from shutil import copyfile from utils import urlPermitted from utils import createInboxQueueDir @@ -1867,11 +1866,6 @@ def runInboxQueue(projectVersion: str, \ 'accounts': {} } - # keep track of the number of queue item read failures - # so that if a file is corrupt then it will eventually - # be ignored rather than endlessly retried - itemReadFailed=0 - heartBeatCtr=0 queueRestoreCtr=0 @@ -1912,24 +1906,19 @@ def runInboxQueue(projectVersion: str, \ print('Loading queue item '+queueFilename) # Load the queue json - try: - with open(queueFilename, 'r') as fp: - queueJson=commentjson.load(fp) - except: - itemReadFailed+=1 - print('WARN: commentjson exception runInboxQueue') - print('WARN: Failed to load inbox queue item '+queueFilename+' (try '+str(itemReadFailed)+')') - if itemReadFailed>4: - # After a few tries we can assume that the file - # is probably corrupt/unreadable - if len(queue)>0: - queue.pop(0) - itemReadFailed=0 - # delete the queue file - if os.path.isfile(queueFilename): + queueJson=loadJson(queueFilename,1) + if not queueJson: + print('WARN: runInboxQueue failed to load inbox queue item '+queueFilename) + # Assume that the file is probably corrupt/unreadable + if len(queue)>0: + queue.pop(0) + # delete the queue file + if os.path.isfile(queueFilename): + try: os.remove(queueFilename) + except: + pass continue - itemReadFailed=0 # clear the daily quotas for maximum numbers of received posts if currTime-quotasLastUpdate>60*60*24: diff --git a/like.py b/like.py index 32e44769..be2e1572 100644 --- a/like.py +++ b/like.py @@ -9,7 +9,6 @@ __status__ = "Production" import os import json import time -import commentjson from pprint import pprint from utils import urlPermitted from utils import getNicknameFromActor diff --git a/manualapprove.py b/manualapprove.py index fb05cb9b..cc0ce824 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -8,7 +8,6 @@ __status__ = "Production" import os import json -import commentjson import time from follow import followedAccountAccepts from follow import followedAccountRejects diff --git a/media.py b/media.py index 112ac9e6..cdb25c44 100644 --- a/media.py +++ b/media.py @@ -12,7 +12,6 @@ import numpy import os import sys import json -import commentjson import datetime from auth import createPassword from shutil import copyfile diff --git a/metadata.py b/metadata.py index 0833abdc..bbf81649 100644 --- a/metadata.py +++ b/metadata.py @@ -8,7 +8,7 @@ __status__ = "Production" import os import json -import commentjson +from utils import loadJson from utils import noOfAccounts from utils import noOfActiveAccountsMonthly @@ -50,14 +50,9 @@ def metaDataInstance(instanceTitle: str, \ if not os.path.isfile(adminActorFilename): return {} - adminActor=None - try: - with open(adminActorFilename, 'r') as fp: - adminActor=commentjson.load(fp) - except: - print('WARN: commentjson exception metaDataInstance') - + adminActor=loadJson(adminActorFilename,0) if not adminActor: + print('WARN: json load exception metaDataInstance') return {} isBot=False diff --git a/person.py b/person.py index 47206ef5..a7f42a50 100644 --- a/person.py +++ b/person.py @@ -8,7 +8,6 @@ __status__ = "Production" import json import time -import commentjson import os import fileinput import subprocess diff --git a/posts.py b/posts.py index 98dc2193..0529fa20 100644 --- a/posts.py +++ b/posts.py @@ -8,7 +8,6 @@ __status__ = "Production" import requests import json -import commentjson import html import datetime import os @@ -2001,11 +2000,9 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \ sharedInboxFilename=os.path.join(sharedBoxDir, postFilename) # get the actor from the shared post - try: - with open(sharedInboxFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) - except: - print('WARN: commentjson exception createSharedInboxIndex') + postJsonObject=loadJson(sharedInboxFilename,0) + if not postJsonObject: + print('WARN: json load exception createSharedInboxIndex') continue actorNickname=getNicknameFromActor(postJsonObject['actor']) @@ -2035,15 +2032,10 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \ continue # read the capabilities id - loadedOcap=False - try: - with open(ocapFilename, 'r') as fp: - ocapJson=commentjson.load(fp) - loadedOcap=True - except: - print('WARN: commentjson exception createSharedInboxIndex') - - if loadedOcap: + ocapJson=loadJson(ocapFilename,0) + if not ocapJson: + print('WARN: json load exception createSharedInboxIndex') + else: if ocapJson.get('id'): if ocapJson['id'] in capsList: postsInBoxDict[statusNumber]=sharedInboxFilename diff --git a/roles.py b/roles.py index 3886b547..282b7aa2 100644 --- a/roles.py +++ b/roles.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import json -import commentjson import os import time from webfinger import webfingerHandle diff --git a/shares.py b/shares.py index 8bf30721..ec025dcc 100644 --- a/shares.py +++ b/shares.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import json -import commentjson import os import time from shutil import copyfile diff --git a/skills.py b/skills.py index 02689b9f..f8afac96 100644 --- a/skills.py +++ b/skills.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import json -import commentjson import os import time from webfinger import webfingerHandle diff --git a/tests.py b/tests.py index f704bf42..b55b70bf 100644 --- a/tests.py +++ b/tests.py @@ -10,7 +10,6 @@ import base64 import time import os, os.path import shutil -import commentjson import json from time import gmtime, strftime from pprint import pprint @@ -41,6 +40,8 @@ from utils import followPerson from utils import getNicknameFromActor from utils import getDomainFromActor from utils import copytree +from utils import loadJson +from utils import saveJson from utils import getStatusNumber from follow import followerOfPerson from follow import unfollowPerson @@ -439,9 +440,8 @@ def testPostMessageBetweenServers(): for name in os.listdir(inboxPath): filename=os.path.join(inboxPath, name) assert os.path.isfile(filename) - receivedJson=None - with open(filename, 'r') as fp: - receivedJson=commentjson.load(fp) + receivedJson=loadJson(filename,0) + if receivedJson: pprint(receivedJson['object']['content']) assert receivedJson assert 'Why is a mouse when it spins?' in receivedJson['object']['content'] @@ -480,8 +480,8 @@ def testPostMessageBetweenServers(): break time.sleep(1) - with open(outboxPostFilename, 'r') as fp: - alicePostJson=commentjson.load(fp) + alicePostJson=loadJson(outboxPostFilename,0) + if alicePostJson: pprint(alicePostJson) assert 'likes' in open(outboxPostFilename).read() @@ -638,8 +638,8 @@ def testFollowBetweenServersWithCapabilities(): break time.sleep(1) - with open(bobCapsFilename, 'r') as fp: - bobCapsJson=commentjson.load(fp) + bobCapsJson=loadJson(bobCapsFilename,0) + if bobCapsJson: if not bobCapsJson.get('capability'): print("Unexpected format for Bob's capabilities") pprint(bobCapsJson) @@ -725,14 +725,16 @@ def testFollowBetweenServersWithCapabilities(): bobPersonCache={} bobCachedWebfingers={} print("Bob's capabilities for Alice:") - with open(bobCapsFilename, 'r') as fp: - bobCapsJson=commentjson.load(fp) + bobCapsJson=loadJson(bobCapsFilename,0) + if bobCapsJson: pprint(bobCapsJson) + assert bobCapsJson.get('capability') assert "inbox:noreply" not in bobCapsJson['capability'] print("Alice's capabilities granted by Bob") - with open(aliceCapsFilename, 'r') as fp: - aliceCapsJson=commentjson.load(fp) + aliceCapsJson=loadJson(aliceCapsFilename,0) + if aliceCapsJson: pprint(aliceCapsJson) + assert aliceCapsJson.get('capability') assert "inbox:noreply" not in aliceCapsJson['capability'] newCapabilities=["inbox:write","objects:read","inbox:noreply"] sendCapabilitiesUpdate(sessionBob,bobDir,httpPrefix, \ @@ -748,8 +750,9 @@ def testFollowBetweenServersWithCapabilities(): bobNewCapsJson=None for i in range(20): time.sleep(1) - with open(bobCapsFilename, 'r') as fp: - bobNewCapsJson=commentjson.load(fp) + bobNewCapsJson=loadJson(bobCapsFilename,0) + if bobNewCapsJson: + assert bobNewCapsJson.get('capability') if "inbox:noreply" in bobNewCapsJson['capability']: print("Bob's capabilities were changed") pprint(bobNewCapsJson) @@ -762,8 +765,9 @@ def testFollowBetweenServersWithCapabilities(): aliceNewCapsJson=None for i in range(20): time.sleep(1) - with open(aliceCapsFilename, 'r') as fp: - aliceNewCapsJson=commentjson.load(fp) + aliceNewCapsJson=loadJson(aliceCapsFilename,0) + if aliceNewCapsJson: + assert aliceNewCapsJson.get('capability') if "inbox:noreply" in aliceNewCapsJson['capability']: print("Alice's granted capabilities were changed") pprint(aliceNewCapsJson) @@ -1367,8 +1371,8 @@ def testClientToServer(): if '#statuses#' in name: statusNumber=int(name.split('#statuses#')[1].replace('.json','').replace('#activity','')) outboxPostFilename=outboxPath+'/'+name - with open(outboxPostFilename, 'r') as fp: - postJsonObject=commentjson.load(fp) + postJsonObject=loadJson(outboxPostFilename,0) + if postJsonObject: outboxPostId=postJsonObject['id'].replace('/activity','') assert outboxPostId print('message id obtained: '+outboxPostId) @@ -1620,11 +1624,8 @@ def testCommentJson() -> None: testJson={ "content": messageStr } - with open(filename, 'w') as fp: - commentjson.dump(testJson, fp, indent=2, sort_keys=False) - receivedJson=None - with open(filename, 'r') as fp: - receivedJson=commentjson.load(fp) + assert saveJson(testJson,filename) + receivedJson=loadJson(filename,0) assert receivedJson assert receivedJson['content']==messageStr encodedStr=json.dumps(testJson, ensure_ascii=False) diff --git a/utils.py b/utils.py index c78fda4a..a3176e0d 100644 --- a/utils.py +++ b/utils.py @@ -27,7 +27,7 @@ def saveJson(jsonObject: {},filename: str) -> bool: tries+=1 return False -def loadJson(filename: str) -> {}: +def loadJson(filename: str,delaySec=2) -> {}: """Makes a few attempts to load a json formatted file """ jsonObject=None @@ -39,7 +39,8 @@ def loadJson(filename: str) -> {}: break except: print('WARN: loadJson exception') - time.sleep(2) + if delaySec>0: + time.sleep(delaySec) tries+=1 return jsonObject diff --git a/webfinger.py b/webfinger.py index 8a1f77e3..cdec770a 100644 --- a/webfinger.py +++ b/webfinger.py @@ -11,7 +11,6 @@ from Crypto.PublicKey import RSA from Crypto.Util import number import requests import json -import commentjson import os import time from session import getJson diff --git a/webinterface.py b/webinterface.py index 50a4d121..c6ec076b 100644 --- a/webinterface.py +++ b/webinterface.py @@ -9,7 +9,6 @@ __status__ = "Production" import json import time import os -import commentjson from collections import OrderedDict from datetime import datetime from datetime import date