From c5204ea1fac6c9b08f9c305892f0a9ed56c0eec6 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 24 Nov 2019 12:12:29 +0000 Subject: [PATCH] Adding json posts to the timeline from cache --- daemon.py | 42 +++++++++++++++-------- inbox.py | 4 +-- person.py | 11 ++++--- posts.py | 88 ++++++++++++++++++++++++++++++------------------- webinterface.py | 2 +- 5 files changed, 93 insertions(+), 54 deletions(-) diff --git a/daemon.py b/daemon.py index 3d5bc376..7708e617 100644 --- a/daemon.py +++ b/daemon.py @@ -2390,7 +2390,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: inboxFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2412,7 +2413,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first inboxFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2465,7 +2467,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: inboxDMFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2487,7 +2490,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first inboxDMFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2540,7 +2544,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: inboxRepliesFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2563,7 +2568,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first inboxRepliesFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2616,7 +2622,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: inboxMediaFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2639,7 +2646,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first inboxMediaFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2731,7 +2739,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: bookmarksFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2753,7 +2762,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first bookmarksFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2801,7 +2811,8 @@ class PubServer(BaseHTTPRequestHandler): # get outbox feed for a person outboxFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir,self.server.domain, \ self.server.port,self.path, \ self.server.httpPrefix, \ @@ -2823,7 +2834,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if a page wasn't specified then show the first one outboxFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2866,7 +2878,8 @@ class PubServer(BaseHTTPRequestHandler): if '/users/' in self.path: if authorized: moderationFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ @@ -2889,7 +2902,8 @@ class PubServer(BaseHTTPRequestHandler): if 'page=' not in self.path: # if no page was specified then show the first moderationFeed= \ - personBoxJson(self.server.session, \ + personBoxJson(self.server.recentPostsCache, \ + self.server.session, \ self.server.baseDir, \ self.server.domain, \ self.server.port, \ diff --git a/inbox.py b/inbox.py index fe1a88be..16881e40 100644 --- a/inbox.py +++ b/inbox.py @@ -64,7 +64,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \ postId=postJsonObject['id'].replace('/activity','').replace('/','#') if recentPostsCache.get('index'): recentPostsCache['index'].append(postId) - recentPostsCache['json'][postId]=postJsonObject.copy() + recentPostsCache['json'][postId]=json.dumps(postJsonObject) recentPostsCache['html'][postId]=htmlStr while len(recentPostsCache['html'].items())>maxRecentPosts: @@ -75,7 +75,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \ recentPostsCache['index']=[postId] recentPostsCache['json']={} recentPostsCache['html']={} - recentPostsCache['json'][postId]=postJsonObject.copy() + recentPostsCache['json'][postId]=json.dumps(postJsonObject) recentPostsCache['html'][postId]=htmlStr def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \ diff --git a/person.py b/person.py index a7f42a50..a67c603a 100644 --- a/person.py +++ b/person.py @@ -404,7 +404,8 @@ def personLookup(domain: str,path: str,baseDir: str) -> {}: # personJson={"user": "unknown"} return personJson -def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \ +def personBoxJson(recentPostsCache: {}, \ + session,baseDir: str,domain: str,port: int,path: str, \ httpPrefix: str,noOfItems: int,boxname: str, \ authorized: bool,ocapAlways: bool) -> []: """Obtain the inbox/outbox/moderation feed for the given person @@ -447,7 +448,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \ if not validNickname(domain,nickname): return None if boxname=='inbox': - return createInbox(session,baseDir,nickname,domain,port,httpPrefix, \ + return createInbox(recentPostsCache, \ + session,baseDir,nickname,domain,port,httpPrefix, \ noOfItems,headerOnly,ocapAlways,pageNumber) elif boxname=='dm': return createDMTimeline(session,baseDir,nickname,domain,port,httpPrefix, \ @@ -469,7 +471,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \ noOfItems,headerOnly,authorized,pageNumber) return None -def personInboxJson(baseDir: str,domain: str,port: int,path: str, \ +def personInboxJson(recentPostsCache: {}, \ + baseDir: str,domain: str,port: int,path: str, \ httpPrefix: str,noOfItems: int,ocapAlways: bool) -> []: """Obtain the inbox feed for the given person Authentication is expected to have already happened @@ -505,7 +508,7 @@ def personInboxJson(baseDir: str,domain: str,port: int,path: str, \ return None if not validNickname(domain,nickname): return None - return createInbox(baseDir,nickname,domain,port,httpPrefix, \ + return createInbox(recentPostsCache,baseDir,nickname,domain,port,httpPrefix, \ noOfItems,headerOnly,ocapAlways,pageNumber) def setDisplayNickname(baseDir: str,nickname: str, domain: str, \ diff --git a/posts.py b/posts.py index 0529fa20..8c8994e2 100644 --- a/posts.py +++ b/posts.py @@ -1782,34 +1782,36 @@ def sendToFollowersThread(session,baseDir: str, \ sendThread.start() return sendThread -def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ - itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \ +def createInbox(recentPostsCache: {}, \ + session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ + itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: + return createBoxIndexed(recentPostsCache, \ + session,baseDir,'inbox',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \ + return createBoxIndexed({},session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'dm',nickname,domain,port,httpPrefix, \ + return createBoxIndexed({},session,baseDir,'dm',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \ + return createBoxIndexed({},session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \ + return createBoxIndexed({},session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}: - return createBoxIndexed(session,baseDir,'outbox',nickname,domain,port,httpPrefix, \ + return createBoxIndexed({},session,baseDir,'outbox',nickname,domain,port,httpPrefix, \ itemsPerPage,headerOnly,authorized,False,pageNumber) def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ @@ -2045,31 +2047,38 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \ postsCtr+=1 return postsCtr -def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool: +def addPostStringToTimeline(postStr: str,boxname: str,postsInBox: [],boxActor: str) -> bool: """ is this a valid timeline post? """ # must be a "Note" or "Announce" type - with open(filePath, 'r') as postFile: - postStr = postFile.read() - if '"Note"' in postStr or '"Announce"' in postStr or \ - ('"Question"' in postStr and '"Create"' in postStr): + if '"Note"' in postStr or '"Announce"' in postStr or \ + ('"Question"' in postStr and '"Create"' in postStr): - if boxname=='dm': - if '#Public' in postStr or '/followers' in postStr: + if boxname=='dm': + if '#Public' in postStr or '/followers' in postStr: + return False + elif boxname=='tlreplies': + if boxActor not in postStr: + return False + elif boxname=='tlmedia': + if '"Create"' in postStr: + if 'mediaType' not in postStr or 'image/' not in postStr: return False - elif boxname=='tlreplies': - if boxActor not in postStr: - return False - elif boxname=='tlmedia': - if '"Create"' in postStr: - if 'mediaType' not in postStr or 'image/' not in postStr: - return False - # add the post to the dictionary - postsInBox.append(postStr) - return True + # add the post to the dictionary + postsInBox.append(postStr) + return True return False -def createBoxIndexed(session,baseDir: str,boxname: str, \ +def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool: + """ Reads a post from file and decides whether it is valid + """ + with open(filePath, 'r') as postFile: + postStr = postFile.read() + return addPostStringToTimeline(postStr,boxname,postsInBox,boxActor) + return False + +def createBoxIndexed(recentPostsCache: {}, \ + session,baseDir: str,boxname: str, \ nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,authorized :bool, \ ocapAlways: bool,pageNumber=None) -> {}: @@ -2158,14 +2167,27 @@ def createBoxIndexed(session,baseDir: str,boxname: str, \ postFilename=postFilename.split('/')[-1] # filename of the post without any extension or path + # This should also correspond to any index entry in the posts cache postUrl=postFilename.replace('\n','').replace('.json','').strip() - # get the full path of the post - fullPostFilename= \ - locatePost(baseDir,nickname,domain,postUrl,False) - if fullPostFilename: - addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor) - else: - print('WARN: unable to locate post '+postUrl) + + postAdded=False + if recentPostsCache.get('index'): + if postUrl in recentPostsCache['index']: + if recentPostsCache['json'].get(postUrl): + addPostStringToTimeline(recentPostsCache['json'][postUrl], \ + boxname,postsInBox,boxActor) + print('Json post added to timeline from cache') + postAdded=True + + if not postAdded: + # get the full path of the post + fullPostFilename= \ + locatePost(baseDir,nickname,domain,postUrl,False) + if fullPostFilename: + addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor) + else: + print('WARN: unable to locate post '+postUrl) + postsCtr+=1 # Generate first and last entries within header diff --git a/webinterface.py b/webinterface.py index fbd4bcf3..629d558f 100644 --- a/webinterface.py +++ b/webinterface.py @@ -1130,7 +1130,7 @@ def htmlProfilePosts(translate: {}, \ currPage=1 while ctr