Remove excess references to commentjson

acceptreject.py

@@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
-import commentjson
 from capabilities import capabilitiesAccept
 from capabilities import capabilitiesGrantedSave
 from utils import getStatusNumber

announce.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import os
 import time
 import json
-import commentjson
 from pprint import pprint
 from utils import getStatusNumber
 from utils import createOutboxDir

availability.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import json
 import time
-import commentjson
 import os
 from webfinger import webfingerHandle
 from auth import createBasicAuthHeader

bookmarks.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import os
 import json
 import time
-import commentjson
 from pprint import pprint
 from utils import urlPermitted
 from utils import getNicknameFromActor

cache.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import os
 import time
 import datetime
-import commentjson
 from utils import loadJson
 from utils import saveJson
         

capabilities.py

@@ -10,7 +10,6 @@ import os
 import datetime
 import time
 import json
-import commentjson
 from auth import createPassword
 from utils import getNicknameFromActor
 from utils import getDomainFromActor

config.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import os
 import time
 import json
-import commentjson
 from utils import loadJson
 from utils import saveJson
 

content.py

@@ -8,9 +8,9 @@ __status__ = "Production"
 
 import os
 import time
-import commentjson
 import email.parser
 from shutil import copyfile
+from utils import loadJson
 
 def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
     """Uses the tags to replace :emoji: with html image markup
@@ -341,21 +341,7 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
                     # will be retained even if default_emoji.json is subsequently updated                    
                     if not os.path.isfile(baseDir+'/emoji/emoji.json'):
                         copyfile(baseDir+'/emoji/default_emoji.json',baseDir+'/emoji/emoji.json')
-                emojiDictCtr=0
-                while not emojiDict and emojiDictCtr<5:
-                    if emojiDictCtr>0:
-                        print('Retry emoji load '+baseDir+'/emoji/emoji.json')
-                    try:
-                        with open(baseDir+'/emoji/emoji.json', 'r') as fp:
-                            emojiDict=commentjson.load(fp)
-                            if emojiDictCtr>0:
-                                print('emojiDict loaded on try '+str(emojiDictCtr))
-                            break
-                    except:
-                        print('WARN: commentjson exception addHtmlTags')
-                        print('Failed to load emoji (try '+str(emojiDictCtr)+'): '+baseDir+'/emoji/emoji.json')
-                        time.sleep(1)
-                        emojiDictCtr+=1
+                emojiDict=loadJson(baseDir+'/emoji/emoji.json')
 
                 #print('TAG: looking up emoji for :'+wordStr2+':')
                 addEmoji(baseDir,':'+wordStr2+':',httpPrefix,originalDomain,replaceEmoji,hashtags,emojiDict)

daemon.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
 #import socketserver
-import commentjson
 import json
 import time
 import base64
@@ -817,12 +816,6 @@ class PubServer(BaseHTTPRequestHandler):
                 timeDiff=int((time.time()-beginSaveTime)*1000)
                 if timeDiff>200:
                     print('SLOW: slow save of inbox queue item '+queueFilename+' took '+str(timeDiff)+' mS')
-                    try:
-                        with open(queueFilename, 'r') as fp:
-                            queueJson=commentjson.load(fp)
-                            print('SLOW: '+str(queueJson))
-                    except:
-                        pass
             self.send_response(201)
             self.end_headers()
             self.server.POSTbusy=False

delete.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import os
 import json
-import commentjson
 from utils import getStatusNumber
 from utils import createOutboxDir
 from utils import urlPermitted

follow.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import json
 import time
-import commentjson
 from pprint import pprint
 import os
 import sys

inbox.py

@@ -11,7 +11,6 @@ import os
 import datetime
 import time
 import json
-import commentjson
 from shutil import copyfile
 from utils import urlPermitted
 from utils import createInboxQueueDir
@@ -1867,11 +1866,6 @@ def runInboxQueue(projectVersion: str, \
         'accounts': {}
     }
 
-    # keep track of the number of queue item read failures
-    # so that if a file is corrupt then it will eventually
-    # be ignored rather than endlessly retried
-    itemReadFailed=0
-
     heartBeatCtr=0
     queueRestoreCtr=0
 
@@ -1912,24 +1906,19 @@ def runInboxQueue(projectVersion: str, \
             print('Loading queue item '+queueFilename)
             
             # Load the queue json
-            try:
-                with open(queueFilename, 'r') as fp:
-                    queueJson=commentjson.load(fp)
-            except:
-                itemReadFailed+=1
-                print('WARN: commentjson exception runInboxQueue')
-                print('WARN: Failed to load inbox queue item '+queueFilename+' (try '+str(itemReadFailed)+')')
-                if itemReadFailed>4:
-                    # After a few tries we can assume that the file
-                    # is probably corrupt/unreadable
-                    if len(queue)>0:
-                        queue.pop(0)
-                    itemReadFailed=0
-                    # delete the queue file
-                    if os.path.isfile(queueFilename):
+            queueJson=loadJson(queueFilename,1)
+            if not queueJson:
+                print('WARN: runInboxQueue failed to load inbox queue item '+queueFilename)
+                # Assume that the file is probably corrupt/unreadable
+                if len(queue)>0:
+                    queue.pop(0)
+                # delete the queue file
+                if os.path.isfile(queueFilename):
+                    try:
                         os.remove(queueFilename)
+                    except:
+                        pass
                 continue
-            itemReadFailed=0
             
             # clear the daily quotas for maximum numbers of received posts
             if currTime-quotasLastUpdate>60*60*24:

like.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import os
 import json
 import time
-import commentjson
 from pprint import pprint
 from utils import urlPermitted
 from utils import getNicknameFromActor

manualapprove.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import os
 import json
-import commentjson
 import time
 from follow import followedAccountAccepts
 from follow import followedAccountRejects

media.py

@@ -12,7 +12,6 @@ import numpy
 import os
 import sys
 import json
-import commentjson
 import datetime
 from auth import createPassword
 from shutil import copyfile

metadata.py

@@ -8,7 +8,7 @@ __status__ = "Production"
 
 import os
 import json
-import commentjson
+from utils import loadJson
 from utils import noOfAccounts
 from utils import noOfActiveAccountsMonthly
 
@@ -50,14 +50,9 @@ def metaDataInstance(instanceTitle: str, \
     if not os.path.isfile(adminActorFilename):
         return {}
 
-    adminActor=None
-    try:
-        with open(adminActorFilename, 'r') as fp:
-            adminActor=commentjson.load(fp)                
-    except:
-        print('WARN: commentjson exception metaDataInstance')
-
+    adminActor=loadJson(adminActorFilename,0)
     if not adminActor:
+        print('WARN: json load exception metaDataInstance')
         return {}
 
     isBot=False

person.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import json
 import time
-import commentjson
 import os
 import fileinput
 import subprocess

posts.py

@@ -8,7 +8,6 @@ __status__ = "Production"
 
 import requests
 import json
-import commentjson
 import html
 import datetime
 import os
@@ -2001,11 +2000,9 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
                 
         sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
         # get the actor from the shared post
-        try:
-            with open(sharedInboxFilename, 'r') as fp:
-                postJsonObject=commentjson.load(fp)                
-        except:
-            print('WARN: commentjson exception createSharedInboxIndex')
+        postJsonObject=loadJson(sharedInboxFilename,0)
+        if not postJsonObject:
+            print('WARN: json load exception createSharedInboxIndex')
             continue
 
         actorNickname=getNicknameFromActor(postJsonObject['actor'])
@@ -2035,15 +2032,10 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
                 continue
 
             # read the capabilities id
-            loadedOcap=False
-            try:
-                with open(ocapFilename, 'r') as fp:
-                    ocapJson=commentjson.load(fp)
-                    loadedOcap=True
-            except:
-                print('WARN: commentjson exception createSharedInboxIndex')
-
-            if loadedOcap:
+            ocapJson=loadJson(ocapFilename,0)
+            if not ocapJson:
+                print('WARN: json load exception createSharedInboxIndex')
+            else:
                 if ocapJson.get('id'):
                     if ocapJson['id'] in capsList:                                    
                         postsInBoxDict[statusNumber]=sharedInboxFilename

roles.py

@@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
-import commentjson
 import os
 import time
 from webfinger import webfingerHandle

shares.py

@@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
-import commentjson
 import os
 import time
 from shutil import copyfile

skills.py

@@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import json
-import commentjson
 import os
 import time
 from webfinger import webfingerHandle

tests.py

@@ -10,7 +10,6 @@ import base64
 import time
 import os, os.path
 import shutil
-import commentjson
 import json
 from time import gmtime, strftime
 from pprint import pprint
@@ -41,6 +40,8 @@ from utils import followPerson
 from utils import getNicknameFromActor
 from utils import getDomainFromActor
 from utils import copytree
+from utils import loadJson
+from utils import saveJson
 from utils import getStatusNumber
 from follow import followerOfPerson
 from follow import unfollowPerson
@@ -439,9 +440,8 @@ def testPostMessageBetweenServers():
     for name in os.listdir(inboxPath):
         filename=os.path.join(inboxPath, name)
         assert os.path.isfile(filename)
-        receivedJson=None
-        with open(filename, 'r') as fp:
-            receivedJson=commentjson.load(fp)
+        receivedJson=loadJson(filename,0)
+        if receivedJson:
             pprint(receivedJson['object']['content'])
         assert receivedJson
         assert 'Why is a mouse when it spins?' in receivedJson['object']['content']
@@ -480,8 +480,8 @@ def testPostMessageBetweenServers():
             break
         time.sleep(1)
 
-    with open(outboxPostFilename, 'r') as fp:
-        alicePostJson=commentjson.load(fp)
+    alicePostJson=loadJson(outboxPostFilename,0)
+    if alicePostJson:
         pprint(alicePostJson)
             
     assert 'likes' in open(outboxPostFilename).read()
@@ -638,8 +638,8 @@ def testFollowBetweenServersWithCapabilities():
                         break
         time.sleep(1)
 
-    with open(bobCapsFilename, 'r') as fp:
-        bobCapsJson=commentjson.load(fp)
+    bobCapsJson=loadJson(bobCapsFilename,0)
+    if bobCapsJson:
         if not bobCapsJson.get('capability'):
             print("Unexpected format for Bob's capabilities")
             pprint(bobCapsJson)
@@ -725,14 +725,16 @@ def testFollowBetweenServersWithCapabilities():
     bobPersonCache={}
     bobCachedWebfingers={}
     print("Bob's capabilities for Alice:")
-    with open(bobCapsFilename, 'r') as fp:
-        bobCapsJson=commentjson.load(fp)
+    bobCapsJson=loadJson(bobCapsFilename,0)
+    if bobCapsJson:
         pprint(bobCapsJson)
+        assert bobCapsJson.get('capability')
         assert "inbox:noreply" not in bobCapsJson['capability']
     print("Alice's capabilities granted by Bob")
-    with open(aliceCapsFilename, 'r') as fp:
-        aliceCapsJson=commentjson.load(fp)
+    aliceCapsJson=loadJson(aliceCapsFilename,0)
+    if aliceCapsJson:
         pprint(aliceCapsJson)
+        assert aliceCapsJson.get('capability')
         assert "inbox:noreply" not in aliceCapsJson['capability']
     newCapabilities=["inbox:write","objects:read","inbox:noreply"]
     sendCapabilitiesUpdate(sessionBob,bobDir,httpPrefix, \
@@ -748,8 +750,9 @@ def testFollowBetweenServersWithCapabilities():
     bobNewCapsJson=None
     for i in range(20):
         time.sleep(1)
-        with open(bobCapsFilename, 'r') as fp:
-            bobNewCapsJson=commentjson.load(fp)
+        bobNewCapsJson=loadJson(bobCapsFilename,0)
+        if bobNewCapsJson:
+            assert bobNewCapsJson.get('capability')
             if "inbox:noreply" in bobNewCapsJson['capability']:
                 print("Bob's capabilities were changed")
                 pprint(bobNewCapsJson)
@@ -762,8 +765,9 @@ def testFollowBetweenServersWithCapabilities():
     aliceNewCapsJson=None
     for i in range(20):
         time.sleep(1)
-        with open(aliceCapsFilename, 'r') as fp:
-            aliceNewCapsJson=commentjson.load(fp)
+        aliceNewCapsJson=loadJson(aliceCapsFilename,0)
+        if aliceNewCapsJson:
+            assert aliceNewCapsJson.get('capability')
             if "inbox:noreply" in aliceNewCapsJson['capability']:
                 print("Alice's granted capabilities were changed")
                 pprint(aliceNewCapsJson)
@@ -1367,8 +1371,8 @@ def testClientToServer():
         if '#statuses#' in name:
             statusNumber=int(name.split('#statuses#')[1].replace('.json','').replace('#activity',''))
             outboxPostFilename=outboxPath+'/'+name
-            with open(outboxPostFilename, 'r') as fp:
-                postJsonObject=commentjson.load(fp)
+            postJsonObject=loadJson(outboxPostFilename,0)
+            if postJsonObject:
                 outboxPostId=postJsonObject['id'].replace('/activity','')
     assert outboxPostId
     print('message id obtained: '+outboxPostId)
@@ -1620,11 +1624,8 @@ def testCommentJson() -> None:
     testJson={
         "content": messageStr
     }
-    with open(filename, 'w') as fp:
-        commentjson.dump(testJson, fp, indent=2, sort_keys=False)
-    receivedJson=None
-    with open(filename, 'r') as fp:
-        receivedJson=commentjson.load(fp)
+    assert saveJson(testJson,filename)
+    receivedJson=loadJson(filename,0)
     assert receivedJson
     assert receivedJson['content']==messageStr
     encodedStr=json.dumps(testJson, ensure_ascii=False)

utils.py

@@ -27,7 +27,7 @@ def saveJson(jsonObject: {},filename: str) -> bool:
             tries+=1
     return False
 
-def loadJson(filename: str) -> {}:
+def loadJson(filename: str,delaySec=2) -> {}:
     """Makes a few attempts to load a json formatted file
     """
     jsonObject=None
@@ -39,7 +39,8 @@ def loadJson(filename: str) -> {}:
                 break
         except:
             print('WARN: loadJson exception')
-            time.sleep(2)
+            if delaySec>0:
+                time.sleep(delaySec)
             tries+=1
     return jsonObject
 

webfinger.py

@@ -11,7 +11,6 @@ from Crypto.PublicKey import RSA
 from Crypto.Util import number
 import requests
 import json
-import commentjson
 import os
 import time
 from session import getJson

webinterface.py

@@ -9,7 +9,6 @@ __status__ = "Production"
 import json
 import time
 import os
-import commentjson
 from collections import OrderedDict
 from datetime import datetime
 from datetime import date