From ff338e4de2e6f310e54f490bfe17ba420fe52d26 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 30 Jun 2019 16:03:26 +0100 Subject: [PATCH] Move webfinger cache --- cache.py | 39 ++++++++++++++++++++++++++++++++ daemon.py | 6 ++--- person.py | 28 ----------------------- posts.py | 63 +++++++++++++++++++++++++++++++++++++++++++++------- webfinger.py | 10 +++++---- 5 files changed, 103 insertions(+), 43 deletions(-) create mode 100644 cache.py diff --git a/cache.py b/cache.py new file mode 100644 index 00000000..9558817d --- /dev/null +++ b/cache.py @@ -0,0 +1,39 @@ +__filename__ = "cache.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "0.0.1" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +# cache of actor json +# If there are repeated lookups then this helps prevent a lot +# of needless network traffic +personCache = {} + +# cached webfinger endpoints +cachedWebfingers = {} + +def storePersonInCache(personUrl: str,personJson) -> None: + """Store an actor in the cache + """ + personCache[personUrl]=personJson + +def storeWebfingerInCache(handle: str,wf) -> None: + """Store a webfinger endpoint in the cache + """ + cachedWebfingers[handle]=wf + +def getPersonFromCache(personUrl: str): + """Get an actor from the cache + """ + if personCache.get(personUrl): + return personCache[personUrl] + return None + +def getWebfingerFromCache(handle: str): + """Get webfinger endpoint from the cache + """ + if cachedWebfingers.get(handle): + return cachedWebfingers[handle] + return None diff --git a/daemon.py b/daemon.py index a2175636..a452e4fc 100644 --- a/daemon.py +++ b/daemon.py @@ -137,10 +137,10 @@ class PubServer(BaseHTTPRequestHandler): self.wfile.write(json.dumps(getPerson).encode('utf-8')) self.GETbusy=False return - getPersonKey = personKeyLookup(thisDomain,self.path) - if getPersonKey: + personKey = personKeyLookup(thisDomain,self.path) + if personKey: self._set_headers('text/html; charset=utf-8') - self.wfile.write(getPersonKey.encode('utf-8')) + self.wfile.write(personKey.encode('utf-8')) self.GETbusy=False return # check that a json file was requested diff --git a/person.py b/person.py index 31a96776..02300b07 100644 --- a/person.py +++ b/person.py @@ -15,34 +15,6 @@ from webfinger import createWebfingerEndpoint from webfinger import storeWebfingerEndpoint from posts import createOutbox -# cache of actor json -# If there are repeated lookups then this helps prevent a lot -# of needless network traffic -personCache = {} - -def storePersonInCache(personUrl: str,personJson) -> None: - personCache[personUrl]=personJson - -def getPersonFromCache(personUrl: str): - if personCache.get(personUrl): - return personCache[personUrl] - return None - -def getPersonKey(username: str,domain: str,keyType='public'): - """Returns the public or private key of a person - """ - handle=username+'@'+domain - baseDir=os.getcwd() - keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key' - if not os.path.isfile(keyFilename): - return '' - keyPem='' - with open(keyFilename, "r") as pemFile: - keyPem=pemFile.read() - if len(keyPem)<20: - return '' - return keyPem - def generateRSAKey() -> (str,str): key = RSA.generate(2048) privateKeyPem = key.exportKey("PEM").decode("utf-8") diff --git a/posts.py b/posts.py index 5711383b..12e0648a 100644 --- a/posts.py +++ b/posts.py @@ -11,15 +11,17 @@ import json import commentjson import html import datetime -import os, shutil +import os +import shutil import threading +import sys +import trace +from cache import storePersonInCache +from cache import getPersonFromCache from pprint import pprint from random import randint from session import getJson from session import postJson -from person import getPersonFromCache -from person import storePersonInCache -from person import getPersonKey try: from BeautifulSoup import BeautifulSoup except ImportError: @@ -31,6 +33,51 @@ sendThreads = [] # stores the results from recent post sending attempts postLog = [] +class threadWithTrace(threading.Thread): + def __init__(self, *args, **keywords): + threading.Thread.__init__(self, *args, **keywords) + self.killed = False + + def start(self): + self.__run_backup = self.run + self.run = self.__run + threading.Thread.start(self) + + def __run(self): + sys.settrace(self.globaltrace) + self.__run_backup() + self.run = self.__run_backup + + def globaltrace(self, frame, event, arg): + if event == 'call': + return self.localtrace + else: + return None + + def localtrace(self, frame, event, arg): + if self.killed: + if event == 'line': + raise SystemExit() + return self.localtrace + + def kill(self): + self.killed = True + +def getPersonKey(username: str,domain: str,keyType='public'): + """Returns the public or private key of a person + """ + handle=username+'@'+domain + baseDir=os.getcwd() + keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key' + if not os.path.isfile(keyFilename): + return '' + keyPem='' + with open(keyFilename, "r") as pemFile: + keyPem=pemFile.read() + if len(keyPem)<20: + return '' + return keyPem + def permitted(url: str,federationList) -> bool: """Is a url from one of the permitted domains? """ @@ -74,7 +121,7 @@ def getPersonBox(session,wfRequest,boxName='inbox'): personUrl = getUserUrl(wfRequest) if not personUrl: return None - personJson=getPersonFromCache(personUrl) + personJson = getPersonFromCache(personUrl) if not personJson: personJson = getJson(session,personUrl,asHeader,None) if not personJson.get(boxName): @@ -307,7 +354,7 @@ def threadSendPost(session,postJsonObject,federationList,inboxUrl: str,signature tries=0 backoffTime=60 for attempt in range(20): - postResult = postJson(session,postJsonObject,federationList,inboxUrl,signatureHeader): + postResult = postJson(session,postJsonObject,federationList,inboxUrl,signatureHeader) if postResult: postLog.append(postJsonObject['published']+' '+postResult+'\n') # keep the length of the log finite @@ -358,9 +405,9 @@ def sendPost(session,username: str, domain: str, toUsername: str, toDomain: str, # Keep the number of threads being used small while len(sendThreads)>10: - sendThreads[0].stop() + sendThreads[0].kill() sendThreads.pop(0) - thr = threading.Thread(target=threadSendPost,args=(session,postJsonObject.copy(),federationList,inboxUrl,signatureHeader.copy()),daemon=True) + thr = threadWithTrace(target=threadSendPost,args=(session,postJsonObject.copy(),federationList,inboxUrl,signatureHeader.copy()),daemon=True) sendThreads.append(thr) thr.start() return 0 diff --git a/webfinger.py b/webfinger.py index 4c835029..1675607e 100644 --- a/webfinger.py +++ b/webfinger.py @@ -14,6 +14,8 @@ import json import commentjson import os from session import getJson +from cache import storeWebfingerInCache +from cache import getWebfingerFromCache def parseHandle(handle): if '.' not in handle: @@ -31,14 +33,14 @@ def parseHandle(handle): return username, domain -cachedWebfingers = {} def webfingerHandle(session,handle: str,https: bool): username, domain = parseHandle(handle) if not username: return None - if cachedWebfingers.get(username+'@'+domain): - return cachedWebfingers[username+'@'+domain] + wf=getWebfingerFromCache(username+'@'+domain) + if wf: + return wf prefix='https' if not https: prefix='http' @@ -50,7 +52,7 @@ def webfingerHandle(session,handle: str,https: bool): #except: # print("Unable to webfinger " + url) # return None - cachedWebfingers[username+'@'+domain] = result + storeWebfingerInCache(username+'@'+domain, result) return result def generateMagicKey(publicKeyPem):