From fd09d2682aec8e4741e5f59b517da638b47de2cf Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 29 Jun 2019 17:47:37 +0100 Subject: [PATCH] paginated outbox --- epicyon.py | 14 +++++++++----- person.py | 19 +++++++++++++++++-- posts.py | 51 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/epicyon.py b/epicyon.py index ffe2fc8a..d0e5389b 100644 --- a/epicyon.py +++ b/epicyon.py @@ -16,6 +16,7 @@ from posts import deleteAllPosts from posts import createOutbox from posts import archivePosts from session import createSession +from session import getJson import json import sys import requests @@ -34,7 +35,10 @@ useTor=False session = createSession(useTor) #asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'} -#userFollowing = getJson(session,"",asHeader,None) +#userFollowing = getJson(session,"https://mastodon.social/users/Gargron/followers?page=true",asHeader,None) +#userFollowing = getJson(session,"https://mastodon.social/users/Gargron/following?page=true",asHeader,None) +#pprint(userFollowing) +#sys.exit() privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(username,domain,https,True) @@ -43,13 +47,13 @@ setPreferredUsername(username,domain,'badger') setBio(username,domain,'Some personal info') #createPublicPost(username, domain, https, "G'day world!", False, True, None, None, 'Not suitable for Vogons') #archivePosts(username,domain,4) -#outboxJson=createOutbox(username,domain,https,3,None) -#pprint(outboxJson) +outboxJson=createOutbox(username,domain,https,2,False,None) +pprint(outboxJson) -runDaemon(domain,port,federationList,useTor) +#runDaemon(domain,port,federationList,useTor) #testHttpsig() -#sys.exit() +sys.exit() #pprint(person) #print('\n') diff --git a/person.py b/person.py index 31948484..f2974d81 100644 --- a/person.py +++ b/person.py @@ -163,6 +163,22 @@ def personLookup(domain: str,path: str) -> {}: def personOutboxJson(domain: str,path: str,https: bool,noOfItems: int) -> []: """Obtain the outbox feed for the given person """ + if not '/outbox' in path: + return None + + # handle page numbers + pageNumber=None + if '?page=' in path: + pageNumber=path.split('?page=')[1] + if pageNumber=='true': + pageNumber=1 + else: + try: + pageNumber=int(pageNumber) + except: + pass + path=path.split('?page=')[0] + if not path.endswith('/outbox'): return None username=None @@ -174,8 +190,7 @@ def personOutboxJson(domain: str,path: str,https: bool,noOfItems: int) -> []: return None if not validUsername(username): return None - startMessageId=None - return createOutbox(username,domain,https,noOfItems,startMessageId) + return createOutbox(username,domain,https,noOfItems,headerOnly,pageNumber) def setPreferredUsername(username: str, domain: str, preferredName: str) -> bool: if len(preferredName)>32: diff --git a/posts.py b/posts.py index e6fad5ae..e9f96357 100644 --- a/posts.py +++ b/posts.py @@ -259,13 +259,19 @@ def createPublicPost(username: str, domain: str, https: bool, content: str, foll commentjson.dump(newPost, fp, indent=4, sort_keys=False) return newPost -def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessageId=None) -> []: +def createOutbox(username: str,domain: str,https: bool,itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}: """Constructs the outbox feed """ prefix='https' if not https: prefix='http' outboxDir = createOutboxDir(username,domain) + pageStr='?page=true' + if pageNumber: + try: + pageStr='?page='+str(pageNumber) + except: + pass outboxHeader = {'@context': 'https://www.w3.org/ns/activitystreams', 'first': prefix+'://'+domain+'/users/'+username+'/outbox?page=true', 'id': prefix+'://'+domain+'/users/'+username+'/outbox', @@ -273,14 +279,14 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa 'totalItems': 0, 'type': 'OrderedCollection'} outboxItems = {'@context': 'https://www.w3.org/ns/activitystreams', - 'id': prefix+'://'+domain+'/users/'+username+'/outbox?page=true', + 'id': prefix+'://'+domain+'/users/'+username+'/outbox'+pageStr, 'orderedItems': [ ], 'partOf': prefix+'://'+domain+'/users/'+username+'/outbox', 'type': 'OrderedCollectionPage'} # counter for posts loop - postCtr=0 + postsOnPageCtr=0 # post filenames sorted in descending order postsInOutbox=sorted(os.listdir(outboxDir), reverse=True) @@ -299,26 +305,30 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa prefix+'://'+domain+'/users/'+username+'/outbox?max_id='+postId+'&page=true' # Insert posts + currPage=1 + postsCtr=0 + if not pageNumber: + pageNumber=1 for postFilename in postsInOutbox: - # Are we at the starting message ID yet? - if startMessageId and prevPostFilename: - if '#statuses#'+startMessageId in postFilename: - # update the prev entry for the last message id - postId = prevPostFilename.split('#statuses#')[1].replace('#activity','') - outboxHeader['prev']= \ - prefix+'://'+domain+'/users/'+username+'/outbox?min_id='+postId+'&page=true' + # Are we at the starting page yet? + if prevPostFilename and currPage==pageNumber and postsCtr==0: + # update the prev entry for the last message id + postId = prevPostFilename.split('#statuses#')[1].replace('#activity','') + outboxHeader['prev']= \ + prefix+'://'+domain+'/users/'+username+'/outbox?min_id='+postId+'&page=true' # get the full path of the post file filePath = os.path.join(outboxDir, postFilename) try: if os.path.isfile(filePath): - if postCtr <= noOfItems: + if currPage == pageNumber and postsOnPageCtr <= itemsPerPage: # get the post as json with open(filePath, 'r') as fp: p=commentjson.load(fp) # insert it into the outbox feed - if postCtr < noOfItems: - outboxItems['orderedItems'].append(p) - elif postCtr == noOfItems: + if postsOnPageCtr < itemsPerPage: + if not headerOnly: + outboxItems['orderedItems'].append(p) + elif postsOnPageCtr == itemsPerPage: # if this is the last post update the next message ID if '/statuses/' in p['id']: postId = p['id'].split('/statuses/')[1].replace('/activity','') @@ -326,14 +336,21 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa prefix+'://'+domain+'/users/'+ \ username+'/outbox?max_id='+ \ postId+'&page=true' - postCtr += 1 + postsOnPageCtr += 1 # remember the last post filename for use with prev prevPostFilename = postFilename - if postCtr > noOfItems: + if postsOnPageCtr > itemsPerPage: break + # count the pages + postsCtr += 1 + if postsCtr >= itemsPerPage: + postsCtr = 0 + currPage += 1 except Exception as e: print(e) - return [outboxHeader,outboxItems] + if headerOnly: + return outboxHeader + return outboxItems def archivePosts(username: str,domain: str,maxPostsInOutbox=256) -> None: """Retain a maximum number of posts within the outbox