mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding json posts to the timeline from cache

main
Bob Mottram 2019-11-24 12:12:29 +00:00
parent dbebc3eaa4
commit c5204ea1fa
5 changed files with 93 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
daemon.py
View File

@ -2390,7 +2390,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
inboxFeed= \ inboxFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2412,7 +2413,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
inboxFeed= \ inboxFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2465,7 +2467,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
inboxDMFeed= \ inboxDMFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2487,7 +2490,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
inboxDMFeed= \ inboxDMFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2540,7 +2544,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
inboxRepliesFeed= \ inboxRepliesFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2563,7 +2568,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
inboxRepliesFeed= \ inboxRepliesFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2616,7 +2622,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
inboxMediaFeed= \ inboxMediaFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2639,7 +2646,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
inboxMediaFeed= \ inboxMediaFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2731,7 +2739,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
bookmarksFeed= \ bookmarksFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2753,7 +2762,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
bookmarksFeed= \ bookmarksFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2801,7 +2811,8 @@ class PubServer(BaseHTTPRequestHandler):
# get outbox feed for a person # get outbox feed for a person
outboxFeed= \ outboxFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir,self.server.domain, \ self.server.baseDir,self.server.domain, \
self.server.port,self.path, \ self.server.port,self.path, \
self.server.httpPrefix, \ self.server.httpPrefix, \
@ -2823,7 +2834,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if a page wasn't specified then show the first one # if a page wasn't specified then show the first one
outboxFeed= \ outboxFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2866,7 +2878,8 @@ class PubServer(BaseHTTPRequestHandler):
if '/users/' in self.path: if '/users/' in self.path:
if authorized: if authorized:
moderationFeed= \ moderationFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \
@ -2889,7 +2902,8 @@ class PubServer(BaseHTTPRequestHandler):
if 'page=' not in self.path: if 'page=' not in self.path:
# if no page was specified then show the first # if no page was specified then show the first
moderationFeed= \ moderationFeed= \
personBoxJson(self.server.session, \ personBoxJson(self.server.recentPostsCache, \
self.server.session, \
self.server.baseDir, \ self.server.baseDir, \
self.server.domain, \ self.server.domain, \
self.server.port, \ self.server.port, \

4
inbox.py
View File

@ -64,7 +64,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
postId=postJsonObject['id'].replace('/activity','').replace('/','#') postId=postJsonObject['id'].replace('/activity','').replace('/','#')
if recentPostsCache.get('index'): if recentPostsCache.get('index'):
recentPostsCache['index'].append(postId) recentPostsCache['index'].append(postId)
recentPostsCache['json'][postId]=postJsonObject.copy() recentPostsCache['json'][postId]=json.dumps(postJsonObject)
recentPostsCache['html'][postId]=htmlStr recentPostsCache['html'][postId]=htmlStr
while len(recentPostsCache['html'].items())>maxRecentPosts: while len(recentPostsCache['html'].items())>maxRecentPosts:
@ -75,7 +75,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
recentPostsCache['index']=[postId] recentPostsCache['index']=[postId]
recentPostsCache['json']={} recentPostsCache['json']={}
recentPostsCache['html']={} recentPostsCache['html']={}
recentPostsCache['json'][postId]=postJsonObject.copy() recentPostsCache['json'][postId]=json.dumps(postJsonObject)
recentPostsCache['html'][postId]=htmlStr recentPostsCache['html'][postId]=htmlStr
def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \ def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \

11
person.py
View File

@ -404,7 +404,8 @@ def personLookup(domain: str,path: str,baseDir: str) -> {}:
# personJson={"user": "unknown"} # personJson={"user": "unknown"}
return personJson return personJson
def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \ def personBoxJson(recentPostsCache: {}, \
session,baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int,boxname: str, \ httpPrefix: str,noOfItems: int,boxname: str, \
authorized: bool,ocapAlways: bool) -> []: authorized: bool,ocapAlways: bool) -> []:
"""Obtain the inbox/outbox/moderation feed for the given person """Obtain the inbox/outbox/moderation feed for the given person
@ -447,7 +448,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \
if not validNickname(domain,nickname): if not validNickname(domain,nickname):
return None return None
if boxname=='inbox': if boxname=='inbox':
return createInbox(session,baseDir,nickname,domain,port,httpPrefix, \ return createInbox(recentPostsCache, \
session,baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,ocapAlways,pageNumber) noOfItems,headerOnly,ocapAlways,pageNumber)
elif boxname=='dm': elif boxname=='dm':
return createDMTimeline(session,baseDir,nickname,domain,port,httpPrefix, \ return createDMTimeline(session,baseDir,nickname,domain,port,httpPrefix, \
@ -469,7 +471,8 @@ def personBoxJson(session,baseDir: str,domain: str,port: int,path: str, \
noOfItems,headerOnly,authorized,pageNumber) noOfItems,headerOnly,authorized,pageNumber)
return None return None
def personInboxJson(baseDir: str,domain: str,port: int,path: str, \ def personInboxJson(recentPostsCache: {}, \
baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int,ocapAlways: bool) -> []: httpPrefix: str,noOfItems: int,ocapAlways: bool) -> []:
"""Obtain the inbox feed for the given person """Obtain the inbox feed for the given person
Authentication is expected to have already happened Authentication is expected to have already happened
@ -505,7 +508,7 @@ def personInboxJson(baseDir: str,domain: str,port: int,path: str, \
return None return None
if not validNickname(domain,nickname): if not validNickname(domain,nickname):
return None return None
return createInbox(baseDir,nickname,domain,port,httpPrefix, \ return createInbox(recentPostsCache,baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,ocapAlways,pageNumber) noOfItems,headerOnly,ocapAlways,pageNumber)
def setDisplayNickname(baseDir: str,nickname: str, domain: str, \ def setDisplayNickname(baseDir: str,nickname: str, domain: str, \

88
posts.py
View File

@ -1782,34 +1782,36 @@ def sendToFollowersThread(session,baseDir: str, \
sendThread.start() sendThread.start()
return sendThread return sendThread
def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createInbox(recentPostsCache: {}, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
return createBoxIndexed(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxIndexed(recentPostsCache, \
session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber) itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxIndexed(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \ return createBoxIndexed({},session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber) itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxIndexed(session,baseDir,'dm',nickname,domain,port,httpPrefix, \ return createBoxIndexed({},session,baseDir,'dm',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber) itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxIndexed(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \ return createBoxIndexed({},session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber) itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxIndexed(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \ return createBoxIndexed({},session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber) itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
return createBoxIndexed(session,baseDir,'outbox',nickname,domain,port,httpPrefix, \ return createBoxIndexed({},session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,authorized,False,pageNumber) itemsPerPage,headerOnly,authorized,False,pageNumber)
def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
@ -2045,31 +2047,38 @@ def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
postsCtr+=1 postsCtr+=1
return postsCtr return postsCtr
def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool: def addPostStringToTimeline(postStr: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
""" is this a valid timeline post? """ is this a valid timeline post?
""" """
# must be a "Note" or "Announce" type # must be a "Note" or "Announce" type
with open(filePath, 'r') as postFile: if '"Note"' in postStr or '"Announce"' in postStr or \
postStr = postFile.read() ('"Question"' in postStr and '"Create"' in postStr):
if '"Note"' in postStr or '"Announce"' in postStr or \
('"Question"' in postStr and '"Create"' in postStr):
if boxname=='dm': if boxname=='dm':
if '#Public' in postStr or '/followers' in postStr: if '#Public' in postStr or '/followers' in postStr:
return False
elif boxname=='tlreplies':
if boxActor not in postStr:
return False
elif boxname=='tlmedia':
if '"Create"' in postStr:
if 'mediaType' not in postStr or 'image/' not in postStr:
return False return False
elif boxname=='tlreplies': # add the post to the dictionary
if boxActor not in postStr: postsInBox.append(postStr)
return False return True
elif boxname=='tlmedia':
if '"Create"' in postStr:
if 'mediaType' not in postStr or 'image/' not in postStr:
return False
# add the post to the dictionary
postsInBox.append(postStr)
return True
return False return False
def createBoxIndexed(session,baseDir: str,boxname: str, \ def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
""" Reads a post from file and decides whether it is valid
"""
with open(filePath, 'r') as postFile:
postStr = postFile.read()
return addPostStringToTimeline(postStr,boxname,postsInBox,boxActor)
return False
def createBoxIndexed(recentPostsCache: {}, \
session,baseDir: str,boxname: str, \
nickname: str,domain: str,port: int,httpPrefix: str, \ nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized :bool, \ itemsPerPage: int,headerOnly: bool,authorized :bool, \
ocapAlways: bool,pageNumber=None) -> {}: ocapAlways: bool,pageNumber=None) -> {}:
@ -2158,14 +2167,27 @@ def createBoxIndexed(session,baseDir: str,boxname: str, \
postFilename=postFilename.split('/')[-1] postFilename=postFilename.split('/')[-1]
# filename of the post without any extension or path # filename of the post without any extension or path
# This should also correspond to any index entry in the posts cache
postUrl=postFilename.replace('\n','').replace('.json','').strip() postUrl=postFilename.replace('\n','').replace('.json','').strip()
# get the full path of the post
fullPostFilename= \ postAdded=False
locatePost(baseDir,nickname,domain,postUrl,False) if recentPostsCache.get('index'):
if fullPostFilename: if postUrl in recentPostsCache['index']:
addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor) if recentPostsCache['json'].get(postUrl):
else: addPostStringToTimeline(recentPostsCache['json'][postUrl], \
print('WARN: unable to locate post '+postUrl) boxname,postsInBox,boxActor)
print('Json post added to timeline from cache')
postAdded=True
if not postAdded:
# get the full path of the post
fullPostFilename= \
locatePost(baseDir,nickname,domain,postUrl,False)
if fullPostFilename:
addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
else:
print('WARN: unable to locate post '+postUrl)
postsCtr+=1 postsCtr+=1
# Generate first and last entries within header # Generate first and last entries within header

2
webinterface.py
View File

@ -1130,7 +1130,7 @@ def htmlProfilePosts(translate: {}, \
currPage=1 currPage=1
while ctr<maxItems and currPage<4: while ctr<maxItems and currPage<4:
outboxFeed= \ outboxFeed= \
personBoxJson(session,baseDir,domain, \ personBoxJson({},session,baseDir,domain, \
port,'/users/'+nickname+'/outbox?page='+str(currPage), \ port,'/users/'+nickname+'/outbox?page='+str(currPage), \
httpPrefix, \ httpPrefix, \
10, 'outbox', \ 10, 'outbox', \