Avoid post conversions between json and string after digest is calculated

daemon.py

@@ -134,7 +134,7 @@ def readFollowList(filename: str):
     return followlist
 
 class PubServer(BaseHTTPRequestHandler):
-    protocol_version = 'HTTP/1.1'
+    protocol_version = 'HTTP/1.0'
 
     def _login_headers(self,fileFormat: str,length: int) -> None:
         self.send_response(200)

httpsig.py

@@ -20,7 +20,7 @@ from time import gmtime, strftime
 from pprint import pprint
 
 def messageContentDigest(messageBodyJsonStr: str) -> str:
-    return base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8')
+    return base64.b64encode(SHA256.new(messageBodyJsonStr.encode('utf-8')).digest()).decode('utf-8')
 
 def signPostHeaders(dateStr: str,privateKeyPem: str, \
                     nickname: str, \
@@ -28,7 +28,7 @@ def signPostHeaders(dateStr: str,privateKeyPem: str, \
                     toDomain: str,toPort: int, \
                     path: str, \
                     httpPrefix: str, \
-                    messageBodyJson: {}) -> str:
+                    messageBodyJsonStr: str) -> str:
     """Returns a raw signature string that can be plugged into a header and
     used to verify the authenticity of an HTTP transmission.
     """
@@ -44,14 +44,13 @@ def signPostHeaders(dateStr: str,privateKeyPem: str, \
 
     if not dateStr:
         dateStr=strftime("%a, %d %b %Y %H:%M:%S %Z", gmtime())
-    keyID = httpPrefix+'://'+domain+'/users/'+nickname+'#main-key'
+    keyID=httpPrefix+'://'+domain+'/users/'+nickname+'#main-key'
-    if not messageBodyJson:
+    if not messageBodyJsonStr:
-        headers = {'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'content-type': 'application/json'}
+        headers={'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'content-type': 'application/json'}
     else:
-        messageBodyJsonStr=json.dumps(messageBodyJson)
         bodyDigest=messageContentDigest(messageBodyJsonStr)
-        headers = {'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': 'application/activity+json'}
+        headers={'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': 'application/activity+json'}
-    privateKeyPem = RSA.import_key(privateKeyPem)
+    privateKeyPem=RSA.import_key(privateKeyPem)
     #headers.update({
     #    '(request-target)': f'post {path}',
     #})
@@ -84,7 +83,7 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \
                        domain: str,port: int, \
                        toDomain: str,toPort: int, \
                        path: str,httpPrefix: str,withDigest: bool, \
-                       messageBodyJson: {}) -> {}:
+                       messageBodyJsonStr: str) -> {}:
     """Note that the domain is the destination, not the sender
     """
     contentType='application/activity+json'
@@ -103,7 +102,6 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \
                             domain,port,toDomain,toPort, \
                             path,httpPrefix,None)
     else:
-        messageBodyJsonStr=json.dumps(messageBodyJson)
         bodyDigest=messageContentDigest(messageBodyJsonStr)
         print('***************************Send (request-target): post '+path)
         print('***************************Send host: '+headerDomain)
@@ -116,7 +114,7 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \
             signPostHeaders(dateStr,privateKeyPem,nickname, \
                             domain,port, \
                             toDomain,toPort, \
-                            path,httpPrefix,messageBodyJson)
+                            path,httpPrefix,messageBodyJsonStr)
     headers['signature'] = signatureHeader
     return headers
 

posts.py

@@ -25,7 +25,7 @@ from pprint import pprint
 from random import randint
 from session import createSession
 from session import getJson
-from session import postJson
+from session import postJsonString
 from session import postImage
 from webfinger import webfingerHandle
 from httpsig import createSignedHeader
@@ -879,7 +879,7 @@ def createReportPost(baseDir: str,
                            True,None, None, subject)
     return postJsonObject
 
-def threadSendPost(session,postJsonObject: {},federationList: [],\
+def threadSendPost(session,postJsonStr: str,federationList: [],\
                    inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
                    debug :bool) -> None:
     """Sends a post with exponential backoff
@@ -887,14 +887,15 @@ def threadSendPost(session,postJsonObject: {},federationList: [],\
     tries=0
     backoffTime=60
     for attempt in range(20):
-        postResult = postJson(session,postJsonObject,federationList, \
+        postResult = \
+            postJsonString(session,postJsonStr,federationList, \
                         inboxUrl,signatureHeaderJson, \
                         "inbox:write")
         if postResult:
             if debug:
                 print('DEBUG: json post to '+inboxUrl+' succeeded')
-            if postJsonObject.get('published'):
+            #if postJsonObject.get('published'):
-                postLog.append(postJsonObject['published']+' '+postResult+'\n')
+            #    postLog.append(postJsonObject['published']+' '+postResult+'\n')
             # keep the length of the log finite
             # Don't accumulate massive files on systems with limited resources
             while len(postLog)>64:
@@ -983,18 +984,22 @@ def sendPost(projectVersion: str, \
     #postPath='/'+inboxUrl.split('/')[-1]
     postPath=inboxUrl.split(toDomain)[1]
 
-    # construct the http header
+    # convert json to string so that there are no
+    # subsequent conversions after creating message body digest
+    postJsonStr=json.dumps(postJsonObject)
+
+    # construct the http header, including the message body digest
     signatureHeaderJson = \
         createSignedHeader(privateKeyPem,nickname,domain,port, \
                            toDomain,toPort, \
-                           postPath,httpPrefix,withDigest,postJsonObject)
+                           postPath,httpPrefix,withDigest,postJsonStr)
 
     # Keep the number of threads being used small
     while len(sendThreads)>10:
         sendThreads[0].kill()
         sendThreads.pop(0)
     thr = threadWithTrace(target=threadSendPost,args=(session, \
-                                                      postJsonObject.copy(), \
+                                                      postJsonStr, \
                                                       federationList, \
                                                       inboxUrl,baseDir, \
                                                       signatureHeaderJson.copy(), \
@@ -1097,7 +1102,7 @@ def sendPostViaServer(projectVersion: str, \
                'Content-type': 'application/json', \
                'Authorization': authHeader}
     postResult = \
-        postJson(session,postJsonObject,[],inboxUrl,headers,"inbox:write")
+        postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write")
     #if not postResult:
     #    if debug:
     #        print('DEBUG: POST failed for c2s to '+inboxUrl)
@@ -1212,11 +1217,15 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \
         return 7
     postPath='/'+inboxUrl.split('/')[-1]
 
-    # construct the http header
+    # convert json to string so that there are no
+    # subsequent conversions after creating message body digest
+    postJsonStr=json.dumps(postJsonObject)
+
+    # construct the http header, including the message body digest
     signatureHeaderJson = \
         createSignedHeader(privateKeyPem,nickname,domain,port, \
                            toDomain,toPort, \
-                           postPath,httpPrefix,withDigest,postJsonObject)
+                           postPath,httpPrefix,withDigest,postJsonStr)
 
     # Keep the number of threads being used small
     while len(sendThreads)>10:
@@ -1227,7 +1236,7 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \
         pprint(postJsonObject)
     thr = threadWithTrace(target=threadSendPost, \
                           args=(session, \
-                                postJsonObject.copy(), \
+                                postJsonStr, \
                                 federationList, \
                                 inboxUrl,baseDir, \
                                 signatureHeaderJson.copy(), \

session.py

@@ -65,6 +65,24 @@ def postJson(session,postJsonObject: {},federationList: [],inboxUrl: str,headers
     postResult = session.post(url = inboxUrl, data = json.dumps(postJsonObject), headers=headers)
     return postResult.text
 
+def postJsonString(session,postJsonStr: str,federationList: [],inboxUrl: str,headers: {},capability: str) -> str:
+    """Post a json message string to the inbox of another person
+    Supplying a capability, such as "inbox:write"
+    NOTE: Here we post a string rather than the original json so that
+    conversions between string and json format don't invalidate
+    the message body digest of http signatures
+    """
+
+    # always allow capability requests
+    if not capability.startswith('cap'):    
+        # check that we are posting to a permitted domain
+        if not urlPermitted(inboxUrl,federationList,capability):
+            print('postJson: '+inboxUrl+' not permitted')
+            return None
+
+    postResult = session.post(url = inboxUrl, data = postJsonStr, headers=headers)
+    return postResult.text
+
 def postImage(session,attachImageFilename: str,federationList: [],inboxUrl: str,headers: {},capability: str) -> str:
     """Post an image to the inbox of another person or outbox via c2s
     Supplying a capability, such as "inbox:write"

tests.py

@@ -18,6 +18,7 @@ from person import createPerson
 from Crypto.Hash import SHA256
 from httpsig import signPostHeaders
 from httpsig import verifyPostHeaders
+from httpsig import messageContentDigest
 from cache import storePersonInCache
 from cache import getPersonFromCache
 from threads import threadWithTrace
@@ -103,14 +104,13 @@ def testHttpsigBase(withDigest):
                             domain, port, \
                             boxpath, httpPrefix, None)
     else:
-        bodyDigest = \
+        bodyDigest = messageContentDigest(messageBodyJsonStr)
-            base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8')
         headers = {'host': headersDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': contentType}
         signatureHeader = \
             signPostHeaders(dateStr,privateKeyPem, nickname, \
                             domain, port, \
                             domain, port, \
-                            boxpath, httpPrefix, messageBodyJson)
+                            boxpath, httpPrefix, messageBodyJsonStr)
 
     headers['signature'] = signatureHeader
     assert verifyPostHeaders(httpPrefix,publicKeyPem,headers, \
@@ -128,7 +128,7 @@ def testHttpsigBase(withDigest):
     else:
         # correct domain but fake message
         messageBodyJsonStr = '{"a key": "a value", "another key": "Fake GNUs", "yet another key": "More Fake GNUs"}'
-        bodyDigest = base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8')
+        bodyDigest = messageContentDigest(messageBodyJsonStr)
         headers = {'host': domain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': contentType}
     headers['signature'] = signatureHeader
     assert verifyPostHeaders(httpPrefix,publicKeyPem,headers, \