From c5204ea1fac6c9b08f9c305892f0a9ed56c0eec6 Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@freedombone.net>
Date: Sun, 24 Nov 2019 12:12:29 +0000
Subject: [PATCH] Adding json posts to the timeline from cache
---
daemon.py | 42 +++++++++++++++--------
inbox.py | 4 +--
person.py | 11 ++++---
posts.py | 88 ++++++++++++++++++++++++++++++-------------------
webinterface.py | 2 +-
5 files changed, 93 insertions(+), 54 deletions(-)
diff --git a/daemon.py b/daemon.py
index 3d5bc376e..7708e6172 100644
--- a/daemon.py
+++ b/daemon.py
@@ -2390,7 +2390,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
inboxFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2412,7 +2413,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
inboxFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2465,7 +2467,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
inboxDMFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2487,7 +2490,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
inboxDMFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2540,7 +2544,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
inboxRepliesFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2563,7 +2568,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
inboxRepliesFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2616,7 +2622,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
inboxMediaFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2639,7 +2646,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
inboxMediaFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2731,7 +2739,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
bookmarksFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2753,7 +2762,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
bookmarksFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2801,7 +2811,8 @@ class PubServer(BaseHTTPRequestHandler):
# get outbox feed for a person
outboxFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir,self.server.domain, \
self.server.port,self.path, \
self.server.httpPrefix, \
@@ -2823,7 +2834,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if a page wasn't specified then show the first one
outboxFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2866,7 +2878,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path:
if authorized:
moderationFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
@@ -2889,7 +2902,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path:
# if no page was specified then show the first
moderationFeed= \
- personBoxJson(self.server.session, \
+ personBoxJson(self.server.recentPostsCache, \
+ self.server.session, \
self.server.baseDir, \
self.server.domain, \
self.server.port, \
diff --git a/inbox.py b/inbox.py
index fe1a88bea..16881e40d 100644
--- a/inbox.py
+++ b/inbox.py
@@ -64,7 +64,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
postId=postJsonObject['id'].replace('/activity','').replace('/','#')
if recentPostsCache.get('index'):
recentPostsCache['index'].append(postId)
- recentPostsCache['json'][postId]=postJsonObject.copy()
+ recentPostsCache['json'][postId]=json.dumps(postJsonObject)
recentPostsCache['html'][postId]=htmlStr
while len(recentPostsCache['html'].items())>maxRecentPosts:
@@ -75,7 +75,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
recentPostsCache['index']=[postId]
recentPostsCache['json']={}
recentPostsCache['html']={}
- recentPostsCache['json'][postId]=postJsonObject.copy()
+ recentPostsCache['json'][postId]=json.dumps(postJsonObject)
recentPostsCache['html'][postId]=htmlStr
def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \
diff --git a/person.py b/person.py
index a7f42a508..a67c603ac 100644
--- a/person.py
+++ b/person.py
@@ -404,7 +404,8 @@ def personLookup(domain: str,path: str,baseDir: str) -> {}:
# personJson={"user": "unknown"}
return personJson
-def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \
+def personBoxJson(recentPostsCache: {}, \
+ session,baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int,boxname: str, \
authorized: bool,ocapAlways: bool) -> []:
"""Obtain the inbox/outbox/moderation feed for the given person
@@ -447,7 +448,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \
if not validNickname(domain,nickname):
return None
if boxname=='inbox':
- return createInbox(session,baseDir,nickname,domain,port,httpPrefix, \
+ return createInbox(recentPostsCache, \
+ session,baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,ocapAlways,pageNumber)
elif boxname=='dm':
return createDMTimeline(session,baseDir,nickname,domain,port,httpPrefix, \
@@ -469,7 +471,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \
noOfItems,headerOnly,authorized,pageNumber)
return None
-def personInboxJson(baseDir: str,domain: str,port: int,path: str, \
+def personInboxJson(recentPostsCache: {}, \
+ baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int,ocapAlways: bool) -> []:
"""Obtain the inbox feed for the given person
Authentication is expected to have already happened
@@ -505,7 +508,7 @@ def personInboxJson(baseDir: str,domain: str,port: int,path: str, \
return None
if not validNickname(domain,nickname):
return None
- return createInbox(baseDir,nickname,domain,port,httpPrefix, \
+ return createInbox(recentPostsCache,baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,ocapAlways,pageNumber)
def setDisplayNickname(baseDir: str,nickname: str, domain: str, \
diff --git a/posts.py b/posts.py
index 0529fa207..8c8994e2f 100644
--- a/posts.py
+++ b/posts.py
@@ -1782,34 +1782,36 @@ def sendToFollowersThread(session,baseDir: str, \
sendThread.start()
return sendThread
-def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
- itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
+def createInbox(recentPostsCache: {}, \
+ session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
+ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
+ return createBoxIndexed(recentPostsCache, \
+ session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
+ return createBoxIndexed({},session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'dm',nickname,domain,port,httpPrefix, \
+ return createBoxIndexed({},session,baseDir,'dm',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
+ return createBoxIndexed({},session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
+ return createBoxIndexed({},session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
- return createBoxIndexed(session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
+ return createBoxIndexed({},session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,authorized,False,pageNumber)
def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
@@ -2045,31 +2047,38 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
postsCtr+=1
return postsCtr
-def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
+def addPostStringToTimeline(postStr: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
""" is this a valid timeline post?
"""
# must be a "Note" or "Announce" type
- with open(filePath, 'r') as postFile:
- postStr = postFile.read()
- if '"Note"' in postStr or '"Announce"' in postStr or \
- ('"Question"' in postStr and '"Create"' in postStr):
+ if '"Note"' in postStr or '"Announce"' in postStr or \
+ ('"Question"' in postStr and '"Create"' in postStr):
- if boxname=='dm':
- if '#Public' in postStr or '/followers' in postStr:
+ if boxname=='dm':
+ if '#Public' in postStr or '/followers' in postStr:
+ return False
+ elif boxname=='tlreplies':
+ if boxActor not in postStr:
+ return False
+ elif boxname=='tlmedia':
+ if '"Create"' in postStr:
+ if 'mediaType' not in postStr or 'image/' not in postStr:
return False
- elif boxname=='tlreplies':
- if boxActor not in postStr:
- return False
- elif boxname=='tlmedia':
- if '"Create"' in postStr:
- if 'mediaType' not in postStr or 'image/' not in postStr:
- return False
- # add the post to the dictionary
- postsInBox.append(postStr)
- return True
+ # add the post to the dictionary
+ postsInBox.append(postStr)
+ return True
return False
-def createBoxIndexed(session,baseDir: str,boxname: str, \
+def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
+ """ Reads a post from file and decides whether it is valid
+ """
+ with open(filePath, 'r') as postFile:
+ postStr = postFile.read()
+ return addPostStringToTimeline(postStr,boxname,postsInBox,boxActor)
+ return False
+
+def createBoxIndexed(recentPostsCache: {}, \
+ session,baseDir: str,boxname: str, \
nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized :bool, \
ocapAlways: bool,pageNumber=None) -> {}:
@@ -2158,14 +2167,27 @@ def createBoxIndexed(session,baseDir: str,boxname: str, \
postFilename=postFilename.split('/')[-1]
# filename of the post without any extension or path
+ # This should also correspond to any index entry in the posts cache
postUrl=postFilename.replace('\n','').replace('.json','').strip()
- # get the full path of the post
- fullPostFilename= \
- locatePost(baseDir,nickname,domain,postUrl,False)
- if fullPostFilename:
- addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
- else:
- print('WARN: unable to locate post '+postUrl)
+
+ postAdded=False
+ if recentPostsCache.get('index'):
+ if postUrl in recentPostsCache['index']:
+ if recentPostsCache['json'].get(postUrl):
+ addPostStringToTimeline(recentPostsCache['json'][postUrl], \
+ boxname,postsInBox,boxActor)
+ print('Json post added to timeline from cache')
+ postAdded=True
+
+ if not postAdded:
+ # get the full path of the post
+ fullPostFilename= \
+ locatePost(baseDir,nickname,domain,postUrl,False)
+ if fullPostFilename:
+ addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
+ else:
+ print('WARN: unable to locate post '+postUrl)
+
postsCtr+=1
# Generate first and last entries within header
diff --git a/webinterface.py b/webinterface.py
index fbd4bcf3c..629d558ff 100644
--- a/webinterface.py
+++ b/webinterface.py
@@ -1130,7 +1130,7 @@ def htmlProfilePosts(translate: {}, \
currPage=1
while ctr<maxItems and currPage<4:
outboxFeed= \
- personBoxJson(session,baseDir,domain, \
+ personBoxJson({},session,baseDir,domain, \
port,'/users/'+nickname+'/outbox?page='+str(currPage), \
httpPrefix, \
10, 'outbox', \