From 848d4387671c3a3d1ca5513304970e1d0862a5db Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 20 Aug 2019 10:37:09 +0100 Subject: [PATCH] Function to expire actor cache entries --- cache.py | 50 ++++++++++++++++++++++++++++++++++--------------- inbox.py | 2 +- posts.py | 2 +- tests.py | 2 +- webinterface.py | 12 ++++++------ 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/cache.py b/cache.py index 9f35cd7c..a127b272 100644 --- a/cache.py +++ b/cache.py @@ -31,26 +31,46 @@ def storePersonInCache(baseDir: str,personUrl: str,personJson: {},personCache: { with open(cacheFilename, 'w') as fp: commentjson.dump(personJson, fp, indent=4, sort_keys=False) +def getPersonFromCache(baseDir: str,personUrl: str,personCache: {}) -> {}: + """Get an actor from the cache + """ + # if the actor is not in memory then try to load it from file + if not personCache.get(personUrl): + cacheFilename=baseDir+'/cache/actors/'+personUrl.replace('/','#')+'.json' + if os.path.isfile(cacheFilename): + with open(cacheFilename, 'r') as fp: + personJson=commentjson.load(fp) + storePersonInCache(baseDir,personUrl,personJson,personCache) + + if personCache.get(personUrl): + # update the timestamp for the last time the actor was retrieved + currTime=datetime.datetime.utcnow() + personCache[personUrl]['timestamp']=currTime.strftime("%Y-%m-%dT%H:%M:%SZ") + return personCache[personUrl]['actor'] + return None + +def expirePersonCache(personCache: {}): + """Expires old entries from the cache in memory + """ + currTime=datetime.datetime.utcnow() + removals=[] + for personUrl,cacheJson in personCache.items(): + cacheTime= \ + datetime.datetime.strptime(cacheJson['timestamp'], \ + "%Y-%m-%dT%H:%M:%SZ") + daysSinceCached=(currTime - cacheTime).days + if daysSinceCached > 2: + removals.append(personUrl) + if len(removals)>0: + for personUrl in removals: + del personCache[personUrl] + print(str(len(removals))+' actors were expired from the cache') + def storeWebfingerInCache(handle: str,wf,cachedWebfingers: {}) -> None: """Store a webfinger endpoint in the cache """ cachedWebfingers[handle]=wf -def getPersonFromCache(personUrl: str,personCache: {}) -> {}: - """Get an actor from the cache - """ - if personCache.get(personUrl): - # how old is the cached data? - currTime=datetime.datetime.utcnow() - cacheTime= \ - datetime.datetime.strptime(personCache[personUrl]['timestamp'], \ - "%Y-%m-%dT%H:%M:%SZ") - daysSinceCached=(currTime - cacheTime).days - # return cached value if it has not expired - if daysSinceCached <= 2: - return personCache[personUrl]['actor'] - return None - def getWebfingerFromCache(handle: str,cachedWebfingers: {}) -> {}: """Get webfinger endpoint from the cache """ diff --git a/inbox.py b/inbox.py index f0ff94e9..e6c9d47e 100644 --- a/inbox.py +++ b/inbox.py @@ -95,7 +95,7 @@ def getPersonPubKey(baseDir: str,session,personUrl: str, \ if debug: print('DEBUG: Obtaining public key for shared inbox') personUrl=personUrl.replace('/users/inbox','/inbox') - personJson = getPersonFromCache(personUrl,personCache) + personJson = getPersonFromCache(baseDir,personUrl,personCache) if not personJson: if debug: print('DEBUG: Obtaining public key for '+personUrl) diff --git a/posts.py b/posts.py index 785267e7..fd2c9d79 100644 --- a/posts.py +++ b/posts.py @@ -146,7 +146,7 @@ def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \ personUrl = getUserUrl(wfRequest) if not personUrl: return None,None,None,None,None,None,None,None - personJson = getPersonFromCache(personUrl,personCache) + personJson = getPersonFromCache(baseDir,personUrl,personCache) if not personJson: personJson = getJson(session,personUrl,asHeader,None, \ projectVersion,httpPrefix,domain) diff --git a/tests.py b/tests.py index 4b3a0f7b..d6bf9034 100644 --- a/tests.py +++ b/tests.py @@ -147,7 +147,7 @@ def testCache(): personJson={ "id": 123456, "test": "This is a test" } personCache={} storePersonInCache(None,personUrl,personJson,personCache) - result=getPersonFromCache(personUrl,personCache) + result=getPersonFromCache(None,personUrl,personCache) assert result['id']==123456 assert result['test']=='This is a test' diff --git a/webinterface.py b/webinterface.py index efb0f46c..8d97aa39 100644 --- a/webinterface.py +++ b/webinterface.py @@ -36,10 +36,10 @@ from config import getConfigParam from skills import getSkills from cache import getPersonFromCache -def getPersonAvatarUrl(personUrl: str,personCache: {}) -> str: +def getPersonAvatarUrl(baseDir: str,personUrl: str,personCache: {}) -> str: """Returns the avatar url for the person """ - personJson = getPersonFromCache(personUrl,personCache) + personJson = getPersonFromCache(baseDir,personUrl,personCache) if personJson: if personJson.get('icon'): if personJson['icon'].get('url'): @@ -947,7 +947,7 @@ def individualFollowAsHtml(baseDir: str,session,wfRequest: {}, \ nickname=getNicknameFromActor(followUrl) domain,port=getDomainFromActor(followUrl) titleStr='@'+nickname+'@'+domain - avatarUrl=getPersonAvatarUrl(followUrl,personCache) + avatarUrl=getPersonAvatarUrl(baseDir,followUrl,personCache) if not avatarUrl: avatarUrl=followUrl+'/avatar.png' if domain not in followUrl: @@ -1093,7 +1093,7 @@ def individualPostAsHtml(baseDir: str, \ attachmentCtr+=1 if not avatarUrl: - avatarUrl=getPersonAvatarUrl(postJsonObject['actor'],personCache) + avatarUrl=getPersonAvatarUrl(baseDir,postJsonObject['actor'],personCache) if not avatarUrl: avatarUrl=postJsonObject['actor']+'/avatar.png' @@ -1321,7 +1321,7 @@ def htmlTimeline(pageNumber: int,itemsPerPage: int,session,baseDir: str, \ for item in timelineJson['orderedItems']: if item['type']=='Create' or item['type']=='Announce': itemCtr+=1 - avatarUrl=getPersonAvatarUrl(item['actor'],personCache) + avatarUrl=getPersonAvatarUrl(baseDir,item['actor'],personCache) tlStr+=individualPostAsHtml(baseDir,session,wfRequest,personCache, \ nickname,domain,port,item,avatarUrl,True, \ allowDeletion, \ @@ -1668,7 +1668,7 @@ def htmlProfileAfterSearch(baseDir: str,path: str,httpPrefix: str, \ if profileJson['icon'].get('url'): avatarUrl=profileJson['icon']['url'] if not avatarUrl: - avatarUrl=getPersonAvatarUrl(personUrl,personCache) + avatarUrl=getPersonAvatarUrl(baseDir,personUrl,personCache) preferredName=searchNickname if profileJson.get('preferredUsername'): preferredName=profileJson['preferredUsername']