paginated outbox

epicyon.py

@@ -16,6 +16,7 @@ from posts import deleteAllPosts
 from posts import createOutbox
 from posts import archivePosts
 from session import createSession
+from session import getJson
 import json
 import sys
 import requests
@@ -34,7 +35,10 @@ useTor=False
 session = createSession(useTor)
 
 #asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
-#userFollowing = getJson(session,"",asHeader,None)
+#userFollowing = getJson(session,"https://mastodon.social/users/Gargron/followers?page=true",asHeader,None)
+#userFollowing = getJson(session,"https://mastodon.social/users/Gargron/following?page=true",asHeader,None)
+#pprint(userFollowing)
+#sys.exit()
 
 
 privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(username,domain,https,True)
@@ -43,13 +47,13 @@ setPreferredUsername(username,domain,'badger')
 setBio(username,domain,'Some personal info')
 #createPublicPost(username, domain, https, "G'day world!", False, True, None, None, 'Not suitable for Vogons')
 #archivePosts(username,domain,4)
-#outboxJson=createOutbox(username,domain,https,3,None)
-#pprint(outboxJson)
+outboxJson=createOutbox(username,domain,https,2,False,None)
+pprint(outboxJson)
 
-runDaemon(domain,port,federationList,useTor)
+#runDaemon(domain,port,federationList,useTor)
 
 #testHttpsig()
-#sys.exit()
+sys.exit()
 
 #pprint(person)
 #print('\n')

person.py

@@ -163,6 +163,22 @@ def personLookup(domain: str,path: str) -> {}:
 def personOutboxJson(domain: str,path: str,https: bool,noOfItems: int) -> []:
     """Obtain the outbox feed for the given person
     """
+    if not '/outbox' in path:
+        return None
+
+    # handle page numbers
+    pageNumber=None
+    if '?page=' in path:
+        pageNumber=path.split('?page=')[1]
+        if pageNumber=='true':
+            pageNumber=1
+        else:
+            try:
+                pageNumber=int(pageNumber)
+            except:
+                pass
+        path=path.split('?page=')[0]
+
     if not path.endswith('/outbox'):
         return None
     username=None
@@ -174,8 +190,7 @@ def personOutboxJson(domain: str,path: str,https: bool,noOfItems: int) -> []:
         return None
     if not validUsername(username):
         return None
-    startMessageId=None
-    return createOutbox(username,domain,https,noOfItems,startMessageId)
+    return createOutbox(username,domain,https,noOfItems,headerOnly,pageNumber)
 
 def setPreferredUsername(username: str, domain: str, preferredName: str) -> bool:
     if len(preferredName)>32:

posts.py

@@ -259,13 +259,19 @@ def createPublicPost(username: str, domain: str, https: bool, content: str, foll
             commentjson.dump(newPost, fp, indent=4, sort_keys=False)
     return newPost
 
-def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessageId=None) -> []:
+def createOutbox(username: str,domain: str,https: bool,itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}:
     """Constructs the outbox feed
     """
     prefix='https'
     if not https:
         prefix='http'
     outboxDir = createOutboxDir(username,domain)
+    pageStr='?page=true'
+    if pageNumber:
+        try:
+            pageStr='?page='+str(pageNumber)
+        except:
+            pass
     outboxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
                     'first': prefix+'://'+domain+'/users/'+username+'/outbox?page=true',
                     'id': prefix+'://'+domain+'/users/'+username+'/outbox',
@@ -273,14 +279,14 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa
                     'totalItems': 0,
                     'type': 'OrderedCollection'}
     outboxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
-                   'id': prefix+'://'+domain+'/users/'+username+'/outbox?page=true',
+                   'id': prefix+'://'+domain+'/users/'+username+'/outbox'+pageStr,
                    'orderedItems': [
                    ],
                    'partOf': prefix+'://'+domain+'/users/'+username+'/outbox',
                    'type': 'OrderedCollectionPage'}
 
     # counter for posts loop
-    postCtr=0
+    postsOnPageCtr=0
 
     # post filenames sorted in descending order
     postsInOutbox=sorted(os.listdir(outboxDir), reverse=True)
@@ -299,26 +305,30 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa
             prefix+'://'+domain+'/users/'+username+'/outbox?max_id='+postId+'&page=true'
 
     # Insert posts
+    currPage=1
+    postsCtr=0
+    if not pageNumber:
+        pageNumber=1
     for postFilename in postsInOutbox:
-        # Are we at the starting message ID yet?
-        if startMessageId and prevPostFilename:
-            if '#statuses#'+startMessageId in postFilename:
-                # update the prev entry for the last message id
-                postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
-                outboxHeader['prev']= \
-                    prefix+'://'+domain+'/users/'+username+'/outbox?min_id='+postId+'&page=true'
+        # Are we at the starting page yet?
+        if prevPostFilename and currPage==pageNumber and postsCtr==0:
+            # update the prev entry for the last message id
+            postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
+            outboxHeader['prev']= \
+                prefix+'://'+domain+'/users/'+username+'/outbox?min_id='+postId+'&page=true'
         # get the full path of the post file
         filePath = os.path.join(outboxDir, postFilename)
         try:
             if os.path.isfile(filePath):
-                if postCtr <= noOfItems:
+                if currPage == pageNumber and postsOnPageCtr <= itemsPerPage:
                     # get the post as json
                     with open(filePath, 'r') as fp:
                         p=commentjson.load(fp)
                         # insert it into the outbox feed
-                        if postCtr < noOfItems:
-                            outboxItems['orderedItems'].append(p)
-                        elif postCtr == noOfItems:
+                        if postsOnPageCtr < itemsPerPage:
+                            if not headerOnly:
+                                outboxItems['orderedItems'].append(p)
+                        elif postsOnPageCtr == itemsPerPage:
                             # if this is the last post update the next message ID
                             if '/statuses/' in p['id']:
                                 postId = p['id'].split('/statuses/')[1].replace('/activity','')
@@ -326,14 +336,21 @@ def createOutbox(username: str,domain: str,https: bool,noOfItems: int,startMessa
                                     prefix+'://'+domain+'/users/'+ \
                                     username+'/outbox?max_id='+ \
                                     postId+'&page=true'
-                        postCtr += 1
+                        postsOnPageCtr += 1
                 # remember the last post filename for use with prev
                 prevPostFilename = postFilename
-                if postCtr > noOfItems:
+                if postsOnPageCtr > itemsPerPage:
                     break
+                # count the pages
+                postsCtr += 1
+                if postsCtr >= itemsPerPage:
+                    postsCtr = 0
+                    currPage += 1
         except Exception as e:
             print(e)
-    return [outboxHeader,outboxItems]
+    if headerOnly:
+        return outboxHeader
+    return outboxItems
 
 def archivePosts(username: str,domain: str,maxPostsInOutbox=256) -> None:
     """Retain a maximum number of posts within the outbox