From 29893507e8b78f6525477015a0dbee5fe553d8bd Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 18 Nov 2019 11:28:17 +0000 Subject: [PATCH] Refactoring timeline generation --- posts.py | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 10 deletions(-) diff --git a/posts.py b/posts.py index 3ace5f151..051804727 100644 --- a/posts.py +++ b/posts.py @@ -1785,28 +1785,28 @@ def sendToFollowersThread(session,baseDir: str, \ def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxBase(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \ - itemsPerPage,headerOnly,True,ocapAlways,pageNumber) + return createBoxIndexed(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \ + itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxBase(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \ - itemsPerPage,headerOnly,True,ocapAlways,pageNumber) + return createBoxIndexed(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \ + itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxBase(session,baseDir,'dm',nickname,domain,port,httpPrefix, \ - itemsPerPage,headerOnly,True,ocapAlways,pageNumber) + return createBoxIndexed(session,baseDir,'dm',nickname,domain,port,httpPrefix, \ + itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxBase(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \ - itemsPerPage,headerOnly,True,ocapAlways,pageNumber) + return createBoxIndexed(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \ + itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}: - return createBoxBase(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \ - itemsPerPage,headerOnly,True,ocapAlways,pageNumber) + return createBoxIndexed(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \ + itemsPerPage,headerOnly,True,ocapAlways,pageNumber) def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}: @@ -2139,6 +2139,7 @@ def createBoxBase(session,baseDir: str,boxname: str, \ fullPostFilename= \ locatePost(baseDir,nickname,domain,postUrl,False) if fullPostFilename: + postsInBox[postsCtr]=fullPostFilename postsCtr+=1 lookedUpFromIndex=True @@ -2275,6 +2276,179 @@ def createBoxBase(session,baseDir: str,boxname: str, \ return boxHeader return boxItems +def isTimelinePost(filePath: str,boxname: str) -> bool: + """ is this a valid timeline post? + """ + # must be a "Note" or "Announce" type + with open(filePath, 'r') as postFile: + postStr = postFile.read() + if '"Note"' in postStr or '"Announce"' in postStr or \ + ('"Question"' in postStr and '"Create"' in postStr): + + if boxname=='dm': + if '#Public' in postStr or '/followers' in postStr: + return False + elif boxname=='tlreplies': + if boxActor not in postStr: + return False + elif boxname=='tlmedia': + if '"Create"' in postStr: + if 'mediaType' not in postStr or 'image/' not in postStr: + return False + return True + return False + +def createBoxIndexed(session,baseDir: str,boxname: str, \ + nickname: str,domain: str,port: int,httpPrefix: str, \ + itemsPerPage: int,headerOnly: bool,authorized :bool, \ + ocapAlways: bool,pageNumber=None) -> {}: + """Constructs the box feed for a person with the given nickname + """ + if not authorized or not pageNumber: + pageNumber=1 + + if boxname!='inbox' and boxname!='dm' and \ + boxname!='tlreplies' and boxname!='tlmedia' and \ + boxname!='outbox' and boxname!='tlbookmarks': + return None + + if boxname!='dm' and boxname!='tlreplies' and \ + boxname!='tlmedia' and boxname!='tlbookmarks': + boxDir = createPersonDir(nickname,domain,baseDir,boxname) + else: + # extract DMs or replies or media from the inbox + boxDir = createPersonDir(nickname,domain,baseDir,'inbox') + + announceCacheDir=baseDir+'/cache/announce/'+nickname + + sharedBoxDir=None + if boxname=='inbox' or boxname=='tlreplies' or \ + boxname=='tlmedia': + sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname) + + # bookmarks timeline is like the inbox but has its own separate index + indexBoxName=boxname + if boxname=='tlbookmarks': + indexBoxName='bookmarks' + + if port: + if port!=80 and port!=443: + if ':' not in domain: + domain=domain+':'+str(port) + + boxActor=httpPrefix+'://'+domain+'/users/'+nickname + + pageStr='?page=true' + if pageNumber: + try: + pageStr='?page='+str(pageNumber) + except: + pass + boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams', + 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true', + 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname, + 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true', + 'totalItems': 0, + 'type': 'OrderedCollection'} + boxItems = {'@context': 'https://www.w3.org/ns/activitystreams', + 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr, + 'orderedItems': [ + ], + 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname, + 'type': 'OrderedCollectionPage'} + + postsInBox=[] + + indexFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+indexBoxName+'.index' + postsCtr=0 + if os.path.isfile(indexFilename): + maxPostCtr=itemsPerPage*pageNumber + with open(indexFilename, 'r') as indexFile: + while postsCtr0: + lastPage=int(postsCtr/itemsPerPage) + if lastPage<1: + lastPage=1 + boxHeader['last']= \ + httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage) + + if headerOnly: + prevPageStr='true' + if pageNumber>1: + prevPageStr=str(pageNumber-1) + boxHeader['prev']= \ + httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+prevPageStr + + nextPageStr='true' + if nextNumber>1: + nextPageStr=str(pageNumber+1) + boxHeader['next']= \ + httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+nextPageStr + return boxHeader + + for postFilename in postsInBox: + if not os.path.isfile(postFilename): + continue + + # get the full path of the post file + filePath = postFilename + p=None + try: + p=json.loads(postStr) + except: + continue + + # remove any capability so that it's not displayed + if p.get('capability'): + del p['capability'] + + # Don't show likes, replies or shares (announces) to unauthorized viewers + if not authorized: + if p.get('object'): + if isinstance(p['object'], dict): + if p['object'].get('likes'): + p['likes']={} + if p['object'].get('replies'): + p['replies']={} + if p['object'].get('shares'): + p['shares']={} + if p['object'].get('bookmarks'): + p['bookmarks']={} + + # insert it into the box feed + if not headerOnly: + boxItems['orderedItems'].append(p) + + return boxItems + def expireCache(baseDir: str,personCache: {},httpPrefix: str,archiveDir: str,maxPostsInBox=32000): """Thread used to expire actors from the cache and archive old posts """