From 22090a1aefda4a0faf640ac53563ba31fedbb602 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Jul 2019 15:30:48 +0100 Subject: [PATCH] Fallbacks for GET and POST locking --- cache.py | 17 +++------- daemon.py | 91 +++++++++++++++++++++++++++++++++++----------------- epicyon.py | 7 ++-- posts.py | 33 ++++++++++++++----- tests.py | 9 ++++-- webfinger.py | 6 ++-- 6 files changed, 104 insertions(+), 59 deletions(-) diff --git a/cache.py b/cache.py index 27e74ad0..239e55a3 100644 --- a/cache.py +++ b/cache.py @@ -8,26 +8,18 @@ __status__ = "Production" import datetime -# cache of actor json -# If there are repeated lookups then this helps prevent a lot -# of needless network traffic -personCache = {} - -# cached webfinger endpoints -cachedWebfingers = {} - -def storePersonInCache(personUrl: str,personJson: {}) -> None: +def storePersonInCache(personUrl: str,personJson: {},personCache: {}) -> None: """Store an actor in the cache """ currTime=datetime.datetime.utcnow() personCache[personUrl]={ "actor": personJson, "timestamp": currTime.strftime("%Y-%m-%dT%H:%M:%SZ") } -def storeWebfingerInCache(handle: str,wf) -> None: +def storeWebfingerInCache(handle: str,wf,cachedWebfingers: {}) -> None: """Store a webfinger endpoint in the cache """ cachedWebfingers[handle]=wf -def getPersonFromCache(personUrl: str) -> {}: +def getPersonFromCache(personUrl: str,personCache: {}) -> {}: """Get an actor from the cache """ if personCache.get(personUrl): @@ -40,10 +32,9 @@ def getPersonFromCache(personUrl: str) -> {}: return personCache[personUrl]['actor'] return None -def getWebfingerFromCache(handle: str) -> {}: +def getWebfingerFromCache(handle: str,cachedWebfingers: {}) -> {}: """Get webfinger endpoint from the cache """ if cachedWebfingers.get(handle): return cachedWebfingers[handle] return None - diff --git a/daemon.py b/daemon.py index 33d2d23e..f3c5673a 100644 --- a/daemon.py +++ b/daemon.py @@ -9,7 +9,7 @@ __status__ = "Production" from http.server import BaseHTTPRequestHandler, HTTPServer #import socketserver import json -import cgi +import time from pprint import pprint from session import createSession from webfinger import webfingerMeta @@ -17,6 +17,7 @@ from webfinger import webfingerLookup from person import personLookup from person import personKeyLookup from person import personOutboxJson +from posts import getPersonPubKey from inbox import inboxPermittedMessage from follow import getFollowingFeed import os @@ -85,59 +86,59 @@ class PubServer(BaseHTTPRequestHandler): return True def do_GET(self): - try: - if self.GETbusy: + if self.server.GETbusy: + currTimeGET=int(time.time()) + if currTimeGET-self.server.lastGET<10: self.send_response(429) self.end_headers() - return - except: - pass - self.GETbusy=True + return + self.server.lastGET=currTimeGET + self.server.GETbusy=True if not self._permittedDir(self.path): self._404() - self.GETbusy=False + self.server.GETbusy=False return # get webfinger endpoint for a person if self._webfinger(): - self.GETbusy=False + self.server.GETbusy=False return # get outbox feed for a person outboxFeed=personOutboxJson(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,maxPostsInFeed) if outboxFeed: self._set_headers('application/json') self.wfile.write(json.dumps(outboxFeed).encode('utf-8')) - self.GETbusy=False + self.server.GETbusy=False return following=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage) if following: self._set_headers('application/json') self.wfile.write(json.dumps(following).encode('utf-8')) - self.GETbusy=False + self.server.GETbusy=False return followers=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage,'followers') if followers: self._set_headers('application/json') self.wfile.write(json.dumps(followers).encode('utf-8')) - self.GETbusy=False + self.server.GETbusy=False return # look up a person getPerson = personLookup(self.server.domain,self.path,self.server.baseDir) if getPerson: self._set_headers('application/json') self.wfile.write(json.dumps(getPerson).encode('utf-8')) - self.GETbusy=False + self.server.GETbusy=False return personKey = personKeyLookup(self.server.domain,self.path,self.server.baseDir) if personKey: self._set_headers('text/html; charset=utf-8') self.wfile.write(personKey.encode('utf-8')) - self.GETbusy=False + self.server.GETbusy=False return # check that a json file was requested if not self.path.endswith('.json'): self._404() - self.GETbusy=False + self.server.GETbusy=False return # check that the file exists filename=self.server.baseDir+self.path @@ -149,27 +150,26 @@ class PubServer(BaseHTTPRequestHandler): self.wfile.write(json.dumps(contentJson).encode('utf8')) else: self._404() - self.GETbusy=False + self.server.GETbusy=False def do_HEAD(self): self._set_headers('application/json') def do_POST(self): - print('**************** POST recieved!') - try: - if self.POSTbusy: + if self.server.POSTbusy: + currTimePOST=int(time.time()) + if currTimePOST-self.server.lastPOST<10: self.send_response(429) self.end_headers() - return - except: - pass + return + self.server.lastPOST=currTimePOST print('**************** POST ready to receive') - self.POSTbusy=True + self.server.POSTbusy=True if not self.headers.get('Content-type'): print('**************** No Content-type') self.send_response(400) self.end_headers() - self.POSTbusy=False + self.server.POSTbusy=False return print('*****************headers: '+str(self.headers)) @@ -178,7 +178,7 @@ class PubServer(BaseHTTPRequestHandler): print("**************** That's no Json!") self.send_response(400) self.end_headers() - self.POSTbusy=False + self.server.POSTbusy=False return # read the message and convert it into a python dictionary @@ -187,7 +187,7 @@ class PubServer(BaseHTTPRequestHandler): if length>maxMessageLength: self.send_response(400) self.end_headers() - self.POSTbusy=False + self.server.POSTbusy=False return print('**************** Reading message') messageBytes=self.rfile.read(length) @@ -197,9 +197,34 @@ class PubServer(BaseHTTPRequestHandler): print('**************** Ah Ah Ah') self.send_response(403) self.end_headers() - self.POSTbusy=False + self.server.POSTbusy=False return + + print('**************** POST get handle') + handle='' + print('**************** POST create session') + currSessionTime=int(time.time()) + if currSessionTime-self.server.sessionLastUpdate>600: + self.server.sessionLastUpdate=currSessionTime + self.server.session = createSession(self.server.useTor) + print('**************** POST webfinger the handle') + wfRequest = webfingerHandle(self.server.session,handle,self.server.https,self.server.cachedWebfingers) + if not wfRequest: + print('**************** POST unknown webfinger') + self.send_response(401) + self.end_headers() + self.server.POSTbusy=False + return + print('**************** POST get public key') + pubKey=getPersonPubKey(self.server.session,wfRequest,self.server.personCache) + if not pubKey: + print('**************** POST no sender public key') + self.send_response(401) + self.end_headers() + self.server.POSTbusy=False + return + print('**************** POST check signature') print('**************** POST valid') pprint(messageJson) # add a property to the object, just to mess with data @@ -210,7 +235,7 @@ class PubServer(BaseHTTPRequestHandler): #self.wfile.write(json.dumps(message).encode('utf-8')) self.send_response(200) self.end_headers() - self.POSTbusy=False + self.server.POSTbusy=False def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None: if len(domain)==0: @@ -218,7 +243,6 @@ def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None: if '.' not in domain: print('Invalid domain: ' + domain) return - session = createSession(useTor) serverAddress = ('', port) httpd = HTTPServer(serverAddress, PubServer) @@ -227,5 +251,14 @@ def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None: httpd.https=https httpd.federationList=fedList.copy() httpd.baseDir=os.getcwd() + httpd.personCache={} + httpd.cachedWebfingers={} + httpd.useTor=useTor + httpd.session = createSession(useTor) + httpd.sessionLastUpdate=int(time.time()) + httpd.lastGET=0 + httpd.lastPOST=0 + httpd.GETbusy=False + httpd.POSTbusy=False print('Running ActivityPub daemon on ' + domain + ' port ' + str(port)) httpd.serve_forever() diff --git a/epicyon.py b/epicyon.py index 16088364..d2b31d00 100644 --- a/epicyon.py +++ b/epicyon.py @@ -47,7 +47,8 @@ https=False useTor=False baseDir=os.getcwd() session = createSession(useTor) - +personCache={} +cachedWebfingers={} clearFollows(baseDir,username,domain) followPerson(baseDir,username,domain,'badger','wild.com',federationList) @@ -94,11 +95,11 @@ sys.exit() #pprint(wfEndpoint) handle="https://mastodon.social/@Gargron" -wfRequest = webfingerHandle(session,handle,True) +wfRequest = webfingerHandle(session,handle,True,cachedWebfingers) if not wfRequest: sys.exit() -personJson,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest) +personJson,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest,personCache) pprint(personJson) sys.exit() diff --git a/posts.py b/posts.py index a46e8720..1b8b3340 100644 --- a/posts.py +++ b/posts.py @@ -82,12 +82,12 @@ def parseUserFeed(session,feedUrl: str,asHeader: {}) -> None: for item in parseUserFeed(session,nextUrl,asHeader): yield item -def getPersonBox(session,wfRequest,boxName='inbox') -> (str,str,str,str): +def getPersonBox(session,wfRequest: {},personCache: {},boxName='inbox') -> (str,str,str,str): asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'} personUrl = getUserUrl(wfRequest) if not personUrl: return None - personJson = getPersonFromCache(personUrl) + personJson = getPersonFromCache(personUrl,personCache) if not personJson: personJson = getJson(session,personUrl,asHeader,None) if not personJson.get(boxName): @@ -103,13 +103,30 @@ def getPersonBox(session,wfRequest,boxName='inbox') -> (str,str,str,str): if personJson['publicKey'].get('publicKeyPem'): pubKey=personJson['publicKey']['publicKeyPem'] - storePersonInCache(personUrl,personJson) + storePersonInCache(personUrl,personJson,personCache) return personJson[boxName],pubKeyId,pubKey,personId -def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: []) -> {}: +def getPersonPubKey(session,wfRequest,personCache: {}) -> str: + asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'} + personUrl = getUserUrl(wfRequest) + if not personUrl: + return None + personJson = getPersonFromCache(personUrl,personCache) + if not personJson: + personJson = getJson(session,personUrl,asHeader,None) + pubKey=None + if personJson.get('publicKey'): + if personJson['publicKey'].get('publicKeyPem'): + pubKey=personJson['publicKey']['publicKeyPem'] + + storePersonInCache(personUrl,personJson,personCache) + + return pubKey + +def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: [],personCache: {}) -> {}: userPosts={} - feedUrl,pubKeyId,pubKey,personId = getPersonBox(session,wfRequest,'outbox') + feedUrl,pubKeyId,pubKey,personId = getPersonBox(session,wfRequest,personCache,'outbox') if not feedUrl: return userPosts @@ -349,7 +366,7 @@ def threadSendPost(session,postJsonObject: {},federationList: [],inboxUrl: str,b time.sleep(backoffTime) backoffTime *= 2 -def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsername: str, toDomain: str, toPort: int, cc: str, https: bool, content: str, followersOnly: bool, saveToFile: bool, federationList: [], sendThreads: [], postLog: [], inReplyTo=None, inReplyToAtomUri=None, subject=None) -> int: +def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsername: str, toDomain: str, toPort: int, cc: str, https: bool, content: str, followersOnly: bool, saveToFile: bool, federationList: [], sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {},inReplyTo=None, inReplyToAtomUri=None, subject=None) -> int: """Post to another inbox """ prefix='https' @@ -364,12 +381,12 @@ def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsern handle=prefix+'://'+toDomain+'/@'+toUsername # lookup the inbox for the To handle - wfRequest = webfingerHandle(session,handle,https) + wfRequest = webfingerHandle(session,handle,https,cachedWebfingers) if not wfRequest: return 1 # get the actor inbox for the To handle - inboxUrl,pubKeyId,pubKey,toPersonId = getPersonBox(session,wfRequest,'inbox') + inboxUrl,pubKeyId,pubKey,toPersonId = getPersonBox(session,wfRequest,personCache,'inbox') if not inboxUrl: return 2 if not pubKey: diff --git a/tests.py b/tests.py index dea78c0f..c0533a80 100644 --- a/tests.py +++ b/tests.py @@ -78,8 +78,9 @@ def testCache(): print('testCache') personUrl="cat@cardboard.box" personJson={ "id": 123456, "test": "This is a test" } - storePersonInCache(personUrl,personJson) - result=getPersonFromCache(personUrl) + personCache={} + storePersonInCache(personUrl,personJson,personCache) + result=getPersonFromCache(personUrl,personCache) assert result['id']==123456 assert result['test']=='This is a test' @@ -189,7 +190,9 @@ def testPostMessageBetweenServers(): followersOnly=False saveToFile=True ccUrl=None - sendResult = sendPost(sessionAlice,aliceDir,'alice', '127.0.0.1', alicePort, 'bob', '127.0.0.1', bobPort, ccUrl, https, 'Why is a mouse when it spins?', followersOnly, saveToFile, federationList, aliceSendThreads, alicePostLog, inReplyTo, inReplyToAtomUri, subject) + alicePersonCache={} + aliceCachedWebfingers={} + sendResult = sendPost(sessionAlice,aliceDir,'alice', '127.0.0.1', alicePort, 'bob', '127.0.0.1', bobPort, ccUrl, https, 'Why is a mouse when it spins?', followersOnly, saveToFile, federationList, aliceSendThreads, alicePostLog, aliceCachedWebfingers,alicePersonCache,inReplyTo, inReplyToAtomUri, subject) print('sendResult: '+str(sendResult)) time.sleep(15) diff --git a/webfinger.py b/webfinger.py index f062b112..08b49144 100644 --- a/webfinger.py +++ b/webfinger.py @@ -34,11 +34,11 @@ def parseHandle(handle: str) -> (str,str): return username, domain -def webfingerHandle(session,handle: str,https: bool) -> {}: +def webfingerHandle(session,handle: str,https: bool,cachedWebfingers: {}) -> {}: username, domain = parseHandle(handle) if not username: return None - wf=getWebfingerFromCache(username+'@'+domain) + wf=getWebfingerFromCache(username+'@'+domain,cachedWebfingers) if wf: return wf prefix='https' @@ -54,7 +54,7 @@ def webfingerHandle(session,handle: str,https: bool) -> {}: #except: # print("Unable to webfinger " + url) # return None - storeWebfingerInCache(username+'@'+domain, result) + storeWebfingerInCache(username+'@'+domain, result,cachedWebfingers) return result def generateMagicKey(publicKeyPem) -> str: