From bd68e84776ae28606922fc14a07cb4d10403ea42 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 24 Nov 2019 11:28:58 +0000 Subject: [PATCH] Store recent posts in memory for fast access --- daemon.py | 33 +++++++++++------ epicyon.py | 4 ++- inbox.py | 58 +++++++++++++++++++++++------- tests.py | 22 ++++++++++-- webinterface.py | 94 ++++++++++++++++++++++++++++++++----------------- 5 files changed, 152 insertions(+), 59 deletions(-) diff --git a/daemon.py b/daemon.py index 9b2a700a9..3d5bc376e 100644 --- a/daemon.py +++ b/daemon.py @@ -2420,7 +2420,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInFeed, 'inbox', \ authorized,self.server.ocapAlways) - msg=htmlInbox(self.server.translate, \ + msg=htmlInbox(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2494,7 +2495,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInFeed, 'dm', \ authorized,self.server.ocapAlways) - msg=htmlInboxDMs(self.server.translate, \ + msg=htmlInboxDMs(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2569,7 +2571,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInFeed, 'tlreplies', \ True,self.server.ocapAlways) - msg=htmlInboxReplies(self.server.translate, \ + msg=htmlInboxReplies(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2644,7 +2647,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInMediaFeed, 'tlmedia', \ True,self.server.ocapAlways) - msg=htmlInboxMedia(self.server.translate, \ + msg=htmlInboxMedia(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInMediaFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2697,7 +2701,8 @@ class PubServer(BaseHTTPRequestHandler): pageNumber=int(pageNumber) else: pageNumber=1 - msg=htmlShares(self.server.translate, \ + msg=htmlShares(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2756,7 +2761,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInFeed, 'tlbookmarks', \ authorized,self.server.ocapAlways) - msg=htmlBookmarks(self.server.translate, \ + msg=htmlBookmarks(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2826,7 +2832,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'outbox', \ authorized, \ self.server.ocapAlways) - msg=htmlOutbox(self.server.translate, \ + msg=htmlOutbox(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -2890,7 +2897,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, \ maxPostsInFeed, 'moderation', \ True,self.server.ocapAlways) - msg=htmlModeration(self.server.translate, \ + msg=htmlModeration(self.server.recentPostsCache, \ + self.server.translate, \ pageNumber,maxPostsInFeed, \ self.server.session, \ self.server.baseDir, \ @@ -5092,7 +5100,8 @@ def loadTokens(baseDir: str,tokensDict: {},tokensLookup: {}) -> None: tokensDict[nickname]=token tokensLookup[token]=nickname -def runDaemon(enableSharedInbox: bool,registration: bool, \ +def runDaemon(maxRecentPosts: int, \ + enableSharedInbox: bool,registration: bool, \ language: str,projectVersion: str, \ instanceId: str,clientToServer: bool, \ baseDir: str,domain: str, \ @@ -5261,10 +5270,14 @@ def runDaemon(enableSharedInbox: bool,registration: bool, \ else: httpd.thrSharesExpire.start() + httpd.recentPostsCache={} + httpd.maxRecentPosts=maxRecentPosts + print('Creating inbox queue') httpd.thrInboxQueue= \ threadWithTrace(target=runInboxQueue, \ - args=(projectVersion, \ + args=(httpd.recentPostsCache,httpd.maxRecentPosts, \ + projectVersion, \ baseDir,httpPrefix,httpd.sendThreads, \ httpd.postLog,httpd.cachedWebfingers, \ httpd.personCache,httpd.inboxQueue, \ diff --git a/epicyon.py b/epicyon.py index ad1e4039b..058442036 100644 --- a/epicyon.py +++ b/epicyon.py @@ -104,6 +104,8 @@ parser.add_argument('-d','--domain', dest='domain', type=str,default=None, \ help='Domain name of the server') parser.add_argument('-p','--port', dest='port', type=int,default=None, \ help='Port number to run on') +parser.add_argument('--postcache', dest='maxRecentPosts', type=int,default=100, \ + help='The maximum number of recent posts to store in RAM') parser.add_argument('--proxy', dest='proxyPort', type=int,default=None, \ help='Proxy port number to run on') parser.add_argument('--path', dest='baseDir', \ @@ -1484,7 +1486,7 @@ if not registration: if setTheme(baseDir,themeName): print('Theme set to '+themeName) -runDaemon(not args.nosharedinbox, \ +runDaemon(args.maxRecentPosts,not args.nosharedinbox, \ registration,args.language,__version__, \ instanceId,args.client,baseDir, \ domain,port,proxyPort,httpPrefix, \ diff --git a/inbox.py b/inbox.py index 6d083af50..fe1a88bea 100644 --- a/inbox.py +++ b/inbox.py @@ -55,7 +55,31 @@ from posts import sendSignedJson from webinterface import individualPostAsHtml from webinterface import getIconsDir -def inboxStorePostToHtmlCache(translate: {}, \ +def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \ + postJsonObject: {},htmlStr: str) -> None: + """Store recent posts in memory so that they can be quickly recalled + """ + if not postJsonObject.get('id'): + return + postId=postJsonObject['id'].replace('/activity','').replace('/','#') + if recentPostsCache.get('index'): + recentPostsCache['index'].append(postId) + recentPostsCache['json'][postId]=postJsonObject.copy() + recentPostsCache['html'][postId]=htmlStr + + while len(recentPostsCache['html'].items())>maxRecentPosts: + recentPostsCache['index'].pop(0) + del recentPostsCache['json'][postId] + del recentPostsCache['html'][postId] + else: + recentPostsCache['index']=[postId] + recentPostsCache['json']={} + recentPostsCache['html']={} + recentPostsCache['json'][postId]=postJsonObject.copy() + recentPostsCache['html'][postId]=htmlStr + +def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \ + translate: {}, \ baseDir: str,httpPrefix: str, \ session,cachedWebfingers: {},personCache: {}, \ nickname: str,domain: str,port: int, \ @@ -68,13 +92,16 @@ def inboxStorePostToHtmlCache(translate: {}, \ showAvatarOptions=True avatarUrl=None boxName='inbox' - individualPostAsHtml(getIconsDir(baseDir),translate,pageNumber, \ - baseDir,session,cachedWebfingers,personCache, \ - nickname,domain,port,postJsonObject, \ - avatarUrl,True,allowDeletion, \ - httpPrefix,__version__,boxName, \ - not isDM(postJsonObject), \ - True,True,False,True) + htmlStr= \ + individualPostAsHtml(getIconsDir(baseDir),translate,pageNumber, \ + baseDir,session,cachedWebfingers,personCache, \ + nickname,domain,port,postJsonObject, \ + avatarUrl,True,allowDeletion, \ + httpPrefix,__version__,boxName, \ + not isDM(postJsonObject), \ + True,True,False,True) + updateRecentPostsCache(recentPostsCache,maxRecentPosts, \ + postJsonObject,htmlStr) def validInbox(baseDir: str,nickname: str,domain: str) -> bool: """Checks whether files were correctly saved to the inbox @@ -1603,7 +1630,8 @@ def inboxUpdateIndex(boxname: str,baseDir: str,handle: str,destinationFilename: return False -def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \ +def inboxAfterCapabilities(recentPostsCache: {},maxRecentPosts: int, \ + session,keyId: str,handle: str,messageJson: {}, \ baseDir: str,httpPrefix: str,sendThreads: [], \ postLog: [],cachedWebfingers: {},personCache: {}, \ queue: [],domain: str,port: int,useTor: bool, \ @@ -1787,7 +1815,8 @@ def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \ if debug: print('DEBUG: saving inbox post as html to cache') htmlCacheStartTime=time.time() - inboxStorePostToHtmlCache(translate,baseDir,httpPrefix, \ + inboxStorePostToHtmlCache(recentPostsCache,maxRecentPosts, \ + translate,baseDir,httpPrefix, \ session,cachedWebfingers,personCache, \ handle.split('@')[0],domain,port, \ postJsonObject,allowDeletion) @@ -1836,7 +1865,8 @@ def runInboxQueueWatchdog(projectVersion: str,httpd) -> None: httpd.thrInboxQueue.start() print('Restarting inbox queue...') -def runInboxQueue(projectVersion: str, \ +def runInboxQueue(recentPostsCache: {},maxRecentPosts: int, \ + projectVersion: str, \ baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \ cachedWebfingers: {},personCache: {},queue: [], \ domain: str,port: int,useTor: bool,federationList: [], \ @@ -2175,7 +2205,8 @@ def runInboxQueue(projectVersion: str, \ # Here the capability id begins with the handle, so this could also # be matched separately, but it's probably not necessary if capsId in capabilityIdList: - inboxAfterCapabilities(session,keyId,handle, \ + inboxAfterCapabilities(recentPostsCache,maxRecentPosts, \ + session,keyId,handle, \ queueJson['post'], \ baseDir,httpPrefix, \ sendThreads,postLog, \ @@ -2194,7 +2225,8 @@ def runInboxQueue(projectVersion: str, \ pprint(queueJson['post']) else: if not ocapAlways: - inboxAfterCapabilities(session,keyId,handle, \ + inboxAfterCapabilities(recentPostsCache,maxRecentPosts, \ + session,keyId,handle, \ queueJson['post'], \ baseDir,httpPrefix, \ sendThreads,postLog, \ diff --git a/tests.py b/tests.py index f521ca417..2c3b8f20c 100644 --- a/tests.py +++ b/tests.py @@ -67,6 +67,7 @@ from media import getAttachmentMediaType from delete import sendDeleteViaServer from inbox import validInbox from inbox import validInboxFilenames +from inbox import updateRecentPostsCache from content import addWebLinks from content import replaceEmojiFromTags from content import addHtmlTags @@ -232,7 +233,7 @@ def createServerAlice(path: str,domain: str,port: int,federationList: [], \ maxMentions=10 maxEmoji=10 print('Server running: Alice') - runDaemon(True,True,'en',__version__, \ + runDaemon(5,True,True,'en',__version__, \ "instanceId",False,path,domain,port,port, \ httpPrefix,federationList,maxMentions,maxEmoji,False, \ noreply,nolike,nopics,noannounce,cw,ocapAlways, \ @@ -288,7 +289,7 @@ def createServerBob(path: str,domain: str,port: int,federationList: [], \ maxMentions=10 maxEmoji=10 print('Server running: Bob') - runDaemon(True,True,'en',__version__, \ + runDaemon(5,True,True,'en',__version__, \ "instanceId",False,path,domain,port,port, \ httpPrefix,federationList,maxMentions,maxEmoji,False, \ noreply,nolike,nopics,noannounce,cw,ocapAlways, \ @@ -324,7 +325,7 @@ def createServerEve(path: str,domain: str,port: int,federationList: [], \ maxMentions=10 maxEmoji=10 print('Server running: Eve') - runDaemon(True,True,'en',__version__, \ + runDaemon(5,True,True,'en',__version__, \ "instanceId",False,path,domain,port,port, \ httpPrefix,federationList,maxMentions,maxEmoji,False, \ noreply,nolike,nopics,noannounce,cw,ocapAlways, \ @@ -1663,8 +1664,23 @@ def testTheme(): result=setCSSparam(css,'background-value','32px') assert result=='--background-value: 32px; --foreground-value: 24px;' +def testRecentPostsCache(): + print('testRecentPostsCache') + recentPostsCache={} + maxRecentPosts=3 + htmlStr='' + for i in range(5): + postJsonObject={ + "id": "https://somesite.whatever/users/someuser/statuses/"+str(i) + } + updateRecentPostsCache(recentPostsCache,maxRecentPosts,postJsonObject,htmlStr) + assert len(recentPostsCache['index'])==maxRecentPosts + assert len(recentPostsCache['json'].items())==maxRecentPosts + assert len(recentPostsCache['html'].items())==maxRecentPosts + def runAllTests(): print('Running tests...') + testRecentPostsCache() testTheme() testSaveLoadJson() testCommentJson() diff --git a/webinterface.py b/webinterface.py index 8979e4647..55f150493 100644 --- a/webinterface.py +++ b/webinterface.py @@ -1876,7 +1876,13 @@ def saveIndividualPostAsHtmlToCache(baseDir: str,nickname: str,domain: str, \ except Exception as e: print('ERROR: saving post to cache '+str(e)) return False - + +def preparePostFromHtmlCache(postHtml: str,boxName: str,pageNumber: int) -> str: + # if on the bookmarks timeline then remain there + if boxName=='tlbookmarks': + postHtml=postHtml.replace('?tl=inbox','?tl=tlbookmarks') + return postHtml.replace(';-999;',';'+str(pageNumber)+';').replace('?page=-999','?page='+str(pageNumber)) + def individualPostAsHtml(iconsDir: str,translate: {}, \ pageNumber: int,baseDir: str, \ session,wfRequest: {},personCache: {}, \ @@ -1908,10 +1914,7 @@ def individualPostAsHtml(iconsDir: str,translate: {}, \ loadIndividualPostAsHtmlFromCache(baseDir,nickname,domain, \ postJsonObject) if postHtml: - # if on the bookmarks timeline then remain there - if boxName=='tlbookmarks': - postHtml=postHtml.replace('?tl=inbox','?tl=tlbookmarks') - return postHtml.replace(';-999;',';'+str(pageNumber)+';').replace('?page=-999','?page='+str(pageNumber)) + return preparePostFromHtmlCache(postHtml,boxName,pageNumber) # don't create new html within the bookmarks timeline # it should already have been created for the inbox @@ -2365,7 +2368,8 @@ def isQuestion(postObjectJson: {}) -> bool: return True return False -def htmlTimeline(translate: {},pageNumber: int, \ +def htmlTimeline(recentPostsCache: {}, \ + translate: {},pageNumber: int, \ itemsPerPage: int,session,baseDir: str, \ wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,timelineJson: {}, \ @@ -2576,16 +2580,26 @@ def htmlTimeline(translate: {},pageNumber: int, \ for item in timelineJson['orderedItems']: if item['type']=='Create' or item['type']=='Announce': #avatarUrl=getPersonAvatarUrl(baseDir,item['actor'],personCache) - currTlStr= \ - individualPostAsHtml(iconsDir,translate,pageNumber, \ - baseDir,session,wfRequest,personCache, \ - nickname,domain,port,item,None,True, \ - allowDeletion, \ - httpPrefix,projectVersion,boxName, \ - boxName!='dm', \ - showIndividualPostIcons, \ - manuallyApproveFollowers,False,True) - + currTlStr=None + if recentPostsCache['index']: + postId=item['id'].replace('/activity','').replace('/','#') + if postId in recentPostsCache['index']: + if recentPostsCache['html'].get(postId): + currTlStr=recentPostsCache['html'][postId] + currTlStr= \ + preparePostFromHtmlCache(currTlStr,boxName,pageNumber) + print('Post obtained from recent cache ('+str(len(recentPostsCache['index']))+'): '+postId) + if not currTlStr: + currTlStr= \ + individualPostAsHtml(iconsDir,translate,pageNumber, \ + baseDir,session,wfRequest,personCache, \ + nickname,domain,port,item,None,True, \ + allowDeletion, \ + httpPrefix,projectVersion,boxName, \ + boxName!='dm', \ + showIndividualPostIcons, \ + manuallyApproveFollowers,False,True) + if currTlStr: itemCtr+=1 tlStr+=currTlStr @@ -2598,7 +2612,8 @@ def htmlTimeline(translate: {},pageNumber: int, \ tlStr+=htmlFooter() return tlStr -def htmlShares(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlShares(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int, \ allowDeletion: bool, \ @@ -2608,12 +2623,14 @@ def htmlShares(translate: {},pageNumber: int,itemsPerPage: int, \ manuallyApproveFollowers= \ followerApprovalActive(baseDir,nickname,domain) - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,None,'tlshares',allowDeletion, \ httpPrefix,projectVersion,manuallyApproveFollowers) -def htmlInbox(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlInbox(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,inboxJson: {}, \ allowDeletion: bool, \ @@ -2623,12 +2640,14 @@ def htmlInbox(translate: {},pageNumber: int,itemsPerPage: int, \ manuallyApproveFollowers= \ followerApprovalActive(baseDir,nickname,domain) - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,inboxJson,'inbox',allowDeletion, \ httpPrefix,projectVersion,manuallyApproveFollowers) -def htmlBookmarks(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlBookmarks(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,bookmarksJson: {}, \ allowDeletion: bool, \ @@ -2638,60 +2657,70 @@ def htmlBookmarks(translate: {},pageNumber: int,itemsPerPage: int, \ manuallyApproveFollowers= \ followerApprovalActive(baseDir,nickname,domain) - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,bookmarksJson,'tlbookmarks',allowDeletion, \ httpPrefix,projectVersion,manuallyApproveFollowers) -def htmlInboxDMs(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlInboxDMs(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,inboxJson: {}, \ allowDeletion: bool, \ httpPrefix: str,projectVersion: str) -> str: """Show the DM timeline as html """ - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,inboxJson,'dm',allowDeletion, \ httpPrefix,projectVersion,False) -def htmlInboxReplies(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlInboxReplies(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,inboxJson: {}, \ allowDeletion: bool, \ httpPrefix: str,projectVersion: str) -> str: """Show the replies timeline as html """ - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,inboxJson,'tlreplies',allowDeletion, \ httpPrefix,projectVersion,False) -def htmlInboxMedia(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlInboxMedia(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,inboxJson: {}, \ allowDeletion: bool, \ httpPrefix: str,projectVersion: str) -> str: """Show the media timeline as html """ - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,inboxJson,'tlmedia',allowDeletion, \ httpPrefix,projectVersion,False) -def htmlModeration(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlModeration(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,inboxJson: {}, \ allowDeletion: bool, \ httpPrefix: str,projectVersion: str) -> str: """Show the moderation feed as html """ - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,inboxJson,'moderation',allowDeletion, \ httpPrefix,projectVersion,True) -def htmlOutbox(translate: {},pageNumber: int,itemsPerPage: int, \ +def htmlOutbox(recentPostsCache: {}, \ + translate: {},pageNumber: int,itemsPerPage: int, \ session,baseDir: str,wfRequest: {},personCache: {}, \ nickname: str,domain: str,port: int,outboxJson: {}, \ allowDeletion: bool, @@ -2700,7 +2729,8 @@ def htmlOutbox(translate: {},pageNumber: int,itemsPerPage: int, \ """ manuallyApproveFollowers= \ followerApprovalActive(baseDir,nickname,domain) - return htmlTimeline(translate,pageNumber, \ + return htmlTimeline(recentPostsCache, \ + translate,pageNumber, \ itemsPerPage,session,baseDir,wfRequest,personCache, \ nickname,domain,port,outboxJson,'outbox',allowDeletion, \ httpPrefix,projectVersion,manuallyApproveFollowers)