From 896b56b3a0f834ac877f7ec7629d60a99577707d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 17 Aug 2019 11:15:01 +0100 Subject: [PATCH] Avoid post conversions between json and string after digest is calculated --- daemon.py | 2 +- httpsig.py | 20 +++++++++----------- posts.py | 41 +++++++++++++++++++++++++---------------- session.py | 18 ++++++++++++++++++ tests.py | 8 ++++---- 5 files changed, 57 insertions(+), 32 deletions(-) diff --git a/daemon.py b/daemon.py index c01e6854..0ac42041 100644 --- a/daemon.py +++ b/daemon.py @@ -134,7 +134,7 @@ def readFollowList(filename: str): return followlist class PubServer(BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' + protocol_version = 'HTTP/1.0' def _login_headers(self,fileFormat: str,length: int) -> None: self.send_response(200) diff --git a/httpsig.py b/httpsig.py index 044a6a03..7e7f4e11 100644 --- a/httpsig.py +++ b/httpsig.py @@ -20,7 +20,7 @@ from time import gmtime, strftime from pprint import pprint def messageContentDigest(messageBodyJsonStr: str) -> str: - return base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8') + return base64.b64encode(SHA256.new(messageBodyJsonStr.encode('utf-8')).digest()).decode('utf-8') def signPostHeaders(dateStr: str,privateKeyPem: str, \ nickname: str, \ @@ -28,7 +28,7 @@ def signPostHeaders(dateStr: str,privateKeyPem: str, \ toDomain: str,toPort: int, \ path: str, \ httpPrefix: str, \ - messageBodyJson: {}) -> str: + messageBodyJsonStr: str) -> str: """Returns a raw signature string that can be plugged into a header and used to verify the authenticity of an HTTP transmission. """ @@ -44,14 +44,13 @@ def signPostHeaders(dateStr: str,privateKeyPem: str, \ if not dateStr: dateStr=strftime("%a, %d %b %Y %H:%M:%S %Z", gmtime()) - keyID = httpPrefix+'://'+domain+'/users/'+nickname+'#main-key' - if not messageBodyJson: - headers = {'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'content-type': 'application/json'} + keyID=httpPrefix+'://'+domain+'/users/'+nickname+'#main-key' + if not messageBodyJsonStr: + headers={'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'content-type': 'application/json'} else: - messageBodyJsonStr=json.dumps(messageBodyJson) bodyDigest=messageContentDigest(messageBodyJsonStr) - headers = {'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': 'application/activity+json'} - privateKeyPem = RSA.import_key(privateKeyPem) + headers={'(request-target)': f'post {path}','host': toDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': 'application/activity+json'} + privateKeyPem=RSA.import_key(privateKeyPem) #headers.update({ # '(request-target)': f'post {path}', #}) @@ -84,7 +83,7 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \ domain: str,port: int, \ toDomain: str,toPort: int, \ path: str,httpPrefix: str,withDigest: bool, \ - messageBodyJson: {}) -> {}: + messageBodyJsonStr: str) -> {}: """Note that the domain is the destination, not the sender """ contentType='application/activity+json' @@ -103,7 +102,6 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \ domain,port,toDomain,toPort, \ path,httpPrefix,None) else: - messageBodyJsonStr=json.dumps(messageBodyJson) bodyDigest=messageContentDigest(messageBodyJsonStr) print('***************************Send (request-target): post '+path) print('***************************Send host: '+headerDomain) @@ -116,7 +114,7 @@ def createSignedHeader(privateKeyPem: str,nickname: str, \ signPostHeaders(dateStr,privateKeyPem,nickname, \ domain,port, \ toDomain,toPort, \ - path,httpPrefix,messageBodyJson) + path,httpPrefix,messageBodyJsonStr) headers['signature'] = signatureHeader return headers diff --git a/posts.py b/posts.py index 5c43dfd3..fd02df4e 100644 --- a/posts.py +++ b/posts.py @@ -25,7 +25,7 @@ from pprint import pprint from random import randint from session import createSession from session import getJson -from session import postJson +from session import postJsonString from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader @@ -879,7 +879,7 @@ def createReportPost(baseDir: str, True,None, None, subject) return postJsonObject -def threadSendPost(session,postJsonObject: {},federationList: [],\ +def threadSendPost(session,postJsonStr: str,federationList: [],\ inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [], debug :bool) -> None: """Sends a post with exponential backoff @@ -887,14 +887,15 @@ def threadSendPost(session,postJsonObject: {},federationList: [],\ tries=0 backoffTime=60 for attempt in range(20): - postResult = postJson(session,postJsonObject,federationList, \ - inboxUrl,signatureHeaderJson, \ - "inbox:write") + postResult = \ + postJsonString(session,postJsonStr,federationList, \ + inboxUrl,signatureHeaderJson, \ + "inbox:write") if postResult: if debug: print('DEBUG: json post to '+inboxUrl+' succeeded') - if postJsonObject.get('published'): - postLog.append(postJsonObject['published']+' '+postResult+'\n') + #if postJsonObject.get('published'): + # postLog.append(postJsonObject['published']+' '+postResult+'\n') # keep the length of the log finite # Don't accumulate massive files on systems with limited resources while len(postLog)>64: @@ -982,19 +983,23 @@ def sendPost(projectVersion: str, \ return 7 #postPath='/'+inboxUrl.split('/')[-1] postPath=inboxUrl.split(toDomain)[1] - - # construct the http header + + # convert json to string so that there are no + # subsequent conversions after creating message body digest + postJsonStr=json.dumps(postJsonObject) + + # construct the http header, including the message body digest signatureHeaderJson = \ createSignedHeader(privateKeyPem,nickname,domain,port, \ toDomain,toPort, \ - postPath,httpPrefix,withDigest,postJsonObject) + postPath,httpPrefix,withDigest,postJsonStr) # Keep the number of threads being used small while len(sendThreads)>10: sendThreads[0].kill() sendThreads.pop(0) thr = threadWithTrace(target=threadSendPost,args=(session, \ - postJsonObject.copy(), \ + postJsonStr, \ federationList, \ inboxUrl,baseDir, \ signatureHeaderJson.copy(), \ @@ -1097,7 +1102,7 @@ def sendPostViaServer(projectVersion: str, \ 'Content-type': 'application/json', \ 'Authorization': authHeader} postResult = \ - postJson(session,postJsonObject,[],inboxUrl,headers,"inbox:write") + postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write") #if not postResult: # if debug: # print('DEBUG: POST failed for c2s to '+inboxUrl) @@ -1211,12 +1216,16 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ print('DEBUG: '+toDomain+' is not in '+inboxUrl) return 7 postPath='/'+inboxUrl.split('/')[-1] - - # construct the http header + + # convert json to string so that there are no + # subsequent conversions after creating message body digest + postJsonStr=json.dumps(postJsonObject) + + # construct the http header, including the message body digest signatureHeaderJson = \ createSignedHeader(privateKeyPem,nickname,domain,port, \ toDomain,toPort, \ - postPath,httpPrefix,withDigest,postJsonObject) + postPath,httpPrefix,withDigest,postJsonStr) # Keep the number of threads being used small while len(sendThreads)>10: @@ -1227,7 +1236,7 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ pprint(postJsonObject) thr = threadWithTrace(target=threadSendPost, \ args=(session, \ - postJsonObject.copy(), \ + postJsonStr, \ federationList, \ inboxUrl,baseDir, \ signatureHeaderJson.copy(), \ diff --git a/session.py b/session.py index 3e1f876a..602f1a26 100644 --- a/session.py +++ b/session.py @@ -65,6 +65,24 @@ def postJson(session,postJsonObject: {},federationList: [],inboxUrl: str,headers postResult = session.post(url = inboxUrl, data = json.dumps(postJsonObject), headers=headers) return postResult.text +def postJsonString(session,postJsonStr: str,federationList: [],inboxUrl: str,headers: {},capability: str) -> str: + """Post a json message string to the inbox of another person + Supplying a capability, such as "inbox:write" + NOTE: Here we post a string rather than the original json so that + conversions between string and json format don't invalidate + the message body digest of http signatures + """ + + # always allow capability requests + if not capability.startswith('cap'): + # check that we are posting to a permitted domain + if not urlPermitted(inboxUrl,federationList,capability): + print('postJson: '+inboxUrl+' not permitted') + return None + + postResult = session.post(url = inboxUrl, data = postJsonStr, headers=headers) + return postResult.text + def postImage(session,attachImageFilename: str,federationList: [],inboxUrl: str,headers: {},capability: str) -> str: """Post an image to the inbox of another person or outbox via c2s Supplying a capability, such as "inbox:write" diff --git a/tests.py b/tests.py index 4047f893..26161a3d 100644 --- a/tests.py +++ b/tests.py @@ -18,6 +18,7 @@ from person import createPerson from Crypto.Hash import SHA256 from httpsig import signPostHeaders from httpsig import verifyPostHeaders +from httpsig import messageContentDigest from cache import storePersonInCache from cache import getPersonFromCache from threads import threadWithTrace @@ -103,14 +104,13 @@ def testHttpsigBase(withDigest): domain, port, \ boxpath, httpPrefix, None) else: - bodyDigest = \ - base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8') + bodyDigest = messageContentDigest(messageBodyJsonStr) headers = {'host': headersDomain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': contentType} signatureHeader = \ signPostHeaders(dateStr,privateKeyPem, nickname, \ domain, port, \ domain, port, \ - boxpath, httpPrefix, messageBodyJson) + boxpath, httpPrefix, messageBodyJsonStr) headers['signature'] = signatureHeader assert verifyPostHeaders(httpPrefix,publicKeyPem,headers, \ @@ -128,7 +128,7 @@ def testHttpsigBase(withDigest): else: # correct domain but fake message messageBodyJsonStr = '{"a key": "a value", "another key": "Fake GNUs", "yet another key": "More Fake GNUs"}' - bodyDigest = base64.b64encode(SHA256.new(messageBodyJsonStr.encode()).digest()).decode('utf-8') + bodyDigest = messageContentDigest(messageBodyJsonStr) headers = {'host': domain,'date': dateStr,'digest': f'SHA-256={bodyDigest}','content-type': contentType} headers['signature'] = signatureHeader assert verifyPostHeaders(httpPrefix,publicKeyPem,headers, \