indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Generic in/outbox functions

master
Bob Mottram 2019-07-04 17:24:23 +01:00
parent e12f0994cf
commit b321baf307
4 changed files with 157 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
daemon.py
View File

@ -16,10 +16,10 @@ from webfinger import webfingerMeta
from webfinger import webfingerLookup from webfinger import webfingerLookup
from webfinger import webfingerHandle from webfinger import webfingerHandle
from person import personLookup from person import personLookup
from person import personOutboxJson from person import personBoxJson
from posts import getPersonPubKey from posts import getPersonPubKey
from posts import outboxMessageCreateWrap from posts import outboxMessageCreateWrap
from posts import savePostToOutbox from posts import savePostToBox
from inbox import inboxPermittedMessage from inbox import inboxPermittedMessage
from inbox import inboxMessageHasParams from inbox import inboxMessageHasParams
from inbox import runInboxQueue from inbox import runInboxQueue
@ -142,7 +142,7 @@ class PubServer(BaseHTTPRequestHandler):
postId=messageJson['id'] postId=messageJson['id']
else: else:
postId=None postId=None
savePostToOutbox(self.server.baseDir,postId,self.postToNickname,self.server.domain,messageJson) savePostToBox(self.server.baseDir,postId,self.postToNickname,self.server.domain,messageJson,'outbox')
return True return True
def do_GET(self): def do_GET(self):
@ -175,13 +175,20 @@ class PubServer(BaseHTTPRequestHandler):
if self.path.endswith('/inbox'): if self.path.endswith('/inbox'):
if '/users/' in self.path: if '/users/' in self.path:
if self.headers.get('Authorization'): if self.headers.get('Authorization'):
if authorize(self.server.baseDir,self.path,self.headers['Authorization'],self.server.debug): if authorize(self.server.baseDir,self.path, \
# TODO self.headers['Authorization'], \
print('inbox access not supported yet') self.server.debug):
self.send_response(405) inboxFeed=personBoxJson(self.server.baseDir, \
self.end_headers() self.server.domain, \
self.server.POSTbusy=False self.server.port, \
return self.path, \
self.server.httpPrefix, \
maxPostsInFeed, 'inbox')
if inboxFeed:
self._set_headers('application/json')
self.wfile.write(json.dumps(inboxFeed).encode('utf-8'))
self.server.GETbusy=False
return
else: else:
if self.server.debug: if self.server.debug:
print('DEBUG: '+nickname+' was not authorized to access '+self.path) print('DEBUG: '+nickname+' was not authorized to access '+self.path)
@ -193,9 +200,10 @@ class PubServer(BaseHTTPRequestHandler):
return return
# get outbox feed for a person # get outbox feed for a person
outboxFeed=personOutboxJson(self.server.baseDir,self.server.domain, \ outboxFeed=personBoxJson(self.server.baseDir,self.server.domain, \
self.server.port,self.path, \ self.server.port,self.path, \
self.server.httpPrefix,maxPostsInFeed) self.server.httpPrefix, \
maxPostsInFeed, 'outbox')
if outboxFeed: if outboxFeed:
self._set_headers('application/json') self._set_headers('application/json')
self.wfile.write(json.dumps(outboxFeed).encode('utf-8')) self.wfile.write(json.dumps(outboxFeed).encode('utf-8'))

59
person.py
View File

@ -148,11 +148,14 @@ def personLookup(domain: str,path: str,baseDir: str) -> {}:
personJson=commentjson.load(fp) personJson=commentjson.load(fp)
return personJson return personJson
def personOutboxJson(baseDir: str,domain: str,port: int,path: str, \ def personBoxJson(baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int) -> []: httpPrefix: str,noOfItems: int,boxname: str) -> []:
"""Obtain the outbox feed for the given person """Obtain the inbox/outbox feed for the given person
""" """
if not '/outbox' in path: if boxname!='inbox' and boxname!='outbox':
return None
if not '/'+boxname in path:
return None return None
# Only show the header by default # Only show the header by default
@ -172,20 +175,62 @@ def personOutboxJson(baseDir: str,domain: str,port: int,path: str, \
path=path.split('?page=')[0] path=path.split('?page=')[0]
headerOnly=False headerOnly=False
if not path.endswith('/outbox'): if not path.endswith('/'+boxname):
return None return None
nickname=None nickname=None
if path.startswith('/users/'): if path.startswith('/users/'):
nickname=path.replace('/users/','',1).replace('/outbox','') nickname=path.replace('/users/','',1).replace('/'+boxname,'')
if path.startswith('/@'): if path.startswith('/@'):
nickname=path.replace('/@','',1).replace('/outbox','') nickname=path.replace('/@','',1).replace('/'+boxname,'')
if not nickname: if not nickname:
return None return None
if not validNickname(nickname): if not validNickname(nickname):
return None return None
if boxname=='inbox':
return createInbox(baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,pageNumber)
return createOutbox(baseDir,nickname,domain,port,httpPrefix, \ return createOutbox(baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,pageNumber) noOfItems,headerOnly,pageNumber)
def personInboxJson(baseDir: str,domain: str,port: int,path: str, \
httpPrefix: str,noOfItems: int) -> []:
"""Obtain the inbox feed for the given person
Authentication is expected to have already happened
"""
if not '/inbox' in path:
return None
# Only show the header by default
headerOnly=True
# handle page numbers
pageNumber=None
if '?page=' in path:
pageNumber=path.split('?page=')[1]
if pageNumber=='true':
pageNumber=1
else:
try:
pageNumber=int(pageNumber)
except:
pass
path=path.split('?page=')[0]
headerOnly=False
if not path.endswith('/inbox'):
return None
nickname=None
if path.startswith('/users/'):
nickname=path.replace('/users/','',1).replace('/inbox','')
if path.startswith('/@'):
nickname=path.replace('/@','',1).replace('/inbox','')
if not nickname:
return None
if not validNickname(nickname):
return None
return createInbox(baseDir,nickname,domain,port,httpPrefix, \
noOfItems,headerOnly,pageNumber)
def setPreferredNickname(baseDir: str,nickname: str, domain: str, \ def setPreferredNickname(baseDir: str,nickname: str, domain: str, \
preferredName: str) -> bool: preferredName: str) -> bool:
if len(preferredName)>32: if len(preferredName)>32:

134
posts.py
View File

@ -28,7 +28,7 @@ from session import postJson
from webfinger import webfingerHandle from webfinger import webfingerHandle
from httpsig import createSignedHeader from httpsig import createSignedHeader
from utils import getStatusNumber from utils import getStatusNumber
from utils import createOutboxDir from utils import createPersonDir
from utils import urlPermitted from utils import urlPermitted
try: try:
from BeautifulSoup import BeautifulSoup from BeautifulSoup import BeautifulSoup
@ -234,23 +234,25 @@ def getPosts(session,outboxUrl: str,maxPosts: int,maxMentions: int, \
break break
return personPosts return personPosts
def createOutboxArchive(nickname: str,domain: str,baseDir: str) -> str: def createBoxArchive(nickname: str,domain: str,baseDir: str,boxname: str) -> str:
"""Creates an archive directory for outbox posts """Creates an archive directory for inbox/outbox posts
""" """
handle=nickname.lower()+'@'+domain.lower() handle=nickname.lower()+'@'+domain.lower()
if not os.path.isdir(baseDir+'/accounts/'+handle): if not os.path.isdir(baseDir+'/accounts/'+handle):
os.mkdir(baseDir+'/accounts/'+handle) os.mkdir(baseDir+'/accounts/'+handle)
outboxArchiveDir=baseDir+'/accounts/'+handle+'/outboxarchive' boxArchiveDir=baseDir+'/accounts/'+handle+'/'+boxname+'archive'
if not os.path.isdir(outboxArchiveDir): if not os.path.isdir(boxArchiveDir):
os.mkdir(outboxArchiveDir) os.mkdir(boxArchiveDir)
return outboxArchiveDir return boxArchiveDir
def deleteAllPosts(baseDir: str,nickname: str, domain: str) -> None: def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
"""Deletes all posts for a person """Deletes all posts for a person from inbox or outbox
""" """
outboxDir = createOutboxDir(nickname,domain,baseDir) if boxname!='inbox' and boxname!='outbox':
for deleteFilename in os.listdir(outboxDir): return
filePath = os.path.join(outboxDir, deleteFilename) boxDir = createPersonDir(nickname,domain,baseDir,boxname)
for deleteFilename in os.listdir(boxDir):
filePath = os.path.join(boxDir, deleteFilename)
try: try:
if os.path.isfile(filePath): if os.path.isfile(filePath):
os.unlink(filePath) os.unlink(filePath)
@ -258,11 +260,14 @@ def deleteAllPosts(baseDir: str,nickname: str, domain: str) -> None:
except Exception as e: except Exception as e:
print(e) print(e)
def savePostToOutbox(baseDir: str,httpPrefix: str,postId: str,nickname: str, domain: str,postJson: {}) -> None: def savePostToBox(baseDir: str,httpPrefix: str,postId: str,nickname: str, domain: str,postJson: {},boxname: str) -> None:
"""Saves the give json to the outbox """Saves the give json to the give box
""" """
if boxname!='inbox' and boxname!='outbox':
return
if ':' in domain: if ':' in domain:
domain=domain.split(':')[0] domain=domain.split(':')[0]
if not postId: if not postId:
statusNumber,published = getStatusNumber() statusNumber,published = getStatusNumber()
postId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber postId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
@ -271,8 +276,8 @@ def savePostToOutbox(baseDir: str,httpPrefix: str,postId: str,nickname: str, dom
postJson['object']['id']=postId postJson['object']['id']=postId
postJson['object']['atomUri']=postId postJson['object']['atomUri']=postId
outboxDir = createOutboxDir(nickname,domain,baseDir) boxDir = createPersonDir(nickname,domain,baseDir,boxname)
filename=outboxDir+'/'+postId.replace('/','#')+'.json' filename=boxDir+'/'+postId.replace('/','#')+'.json'
with open(filename, 'w') as fp: with open(filename, 'w') as fp:
commentjson.dump(postJson, fp, indent=4, sort_keys=False) commentjson.dump(postJson, fp, indent=4, sort_keys=False)
@ -366,7 +371,7 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
newPost['cc']=ccUrl newPost['cc']=ccUrl
newPost['object']['cc']=ccUrl newPost['object']['cc']=ccUrl
if saveToFile: if saveToFile:
savePostToOutbox(baseDir,httpPrefix,newPostId,nickname,domain,newPost) savePostToBox(baseDir,httpPrefix,newPostId,nickname,domain,newPost,'outbox')
return newPost return newPost
def outboxMessageCreateWrap(httpPrefix: str,nickname: str,domain: str,messageJson: {}) -> {}: def outboxMessageCreateWrap(httpPrefix: str,nickname: str,domain: str,messageJson: {}) -> {}:
@ -499,11 +504,22 @@ def sendPost(session,baseDir: str,nickname: str, domain: str, port: int, \
thr.start() thr.start()
return 0 return 0
def createInbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}:
return createBoxBase(baseDir,'inbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,pageNumber)
def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}: itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}:
"""Constructs the outbox feed return createBoxBase(baseDir,'outbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,pageNumber)
def createBoxBase(baseDir: str,boxname: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}:
"""Constructs the box feed
""" """
outboxDir = createOutboxDir(nickname,domain,baseDir) if boxname!='inbox' and boxname!='outbox':
return None
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
if port!=80 and port!=443: if port!=80 and port!=443:
domain = domain+':'+str(port) domain = domain+':'+str(port)
@ -514,69 +530,69 @@ def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: st
pageStr='?page='+str(pageNumber) pageStr='?page='+str(pageNumber)
except: except:
pass pass
outboxHeader = {'@context': 'https://www.w3.org/ns/activitystreams', boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
'first': httpPrefix+'://'+domain+'/users/'+nickname+'/outbox?page=true', 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/outbox', 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'last': httpPrefix+'://'+domain+'/users/'+nickname+'/outbox?page=true', 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'totalItems': 0, 'totalItems': 0,
'type': 'OrderedCollection'} 'type': 'OrderedCollection'}
outboxItems = {'@context': 'https://www.w3.org/ns/activitystreams', boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/outbox'+pageStr, 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
'orderedItems': [ 'orderedItems': [
], ],
'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/outbox', 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'type': 'OrderedCollectionPage'} 'type': 'OrderedCollectionPage'}
# counter for posts loop # counter for posts loop
postsOnPageCtr=0 postsOnPageCtr=0
# post filenames sorted in descending order # post filenames sorted in descending order
postsInOutbox=sorted(os.listdir(outboxDir), reverse=True) postsInBox=sorted(os.listdir(boxDir), reverse=True)
# number of posts in outbox # number of posts in box
outboxHeader['totalItems']=len(postsInOutbox) boxHeader['totalItems']=len(postsInBox)
prevPostFilename=None prevPostFilename=None
if not pageNumber: if not pageNumber:
pageNumber=1 pageNumber=1
# Generate first and last entries within header # Generate first and last entries within header
if len(postsInOutbox)>0: if len(postsInBox)>0:
lastPage=int(len(postsInOutbox)/itemsPerPage) lastPage=int(len(postsInBox)/itemsPerPage)
if lastPage<1: if lastPage<1:
lastPage=1 lastPage=1
outboxHeader['last']= \ boxHeader['last']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/outbox?page='+str(lastPage) httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
# Insert posts # Insert posts
currPage=1 currPage=1
postsCtr=0 postsCtr=0
for postFilename in postsInOutbox: for postFilename in postsInBox:
# Are we at the starting page yet? # Are we at the starting page yet?
if prevPostFilename and currPage==pageNumber and postsCtr==0: if prevPostFilename and currPage==pageNumber and postsCtr==0:
# update the prev entry for the last message id # update the prev entry for the last message id
postId = prevPostFilename.split('#statuses#')[1].replace('#activity','') postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
outboxHeader['prev']= \ boxHeader['prev']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/outbox?min_id='+postId+'&page=true' httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?min_id='+postId+'&page=true'
# get the full path of the post file # get the full path of the post file
filePath = os.path.join(outboxDir, postFilename) filePath = os.path.join(boxDir, postFilename)
try: try:
if os.path.isfile(filePath): if os.path.isfile(filePath):
if currPage == pageNumber and postsOnPageCtr <= itemsPerPage: if currPage == pageNumber and postsOnPageCtr <= itemsPerPage:
# get the post as json # get the post as json
with open(filePath, 'r') as fp: with open(filePath, 'r') as fp:
p=commentjson.load(fp) p=commentjson.load(fp)
# insert it into the outbox feed # insert it into the box feed
if postsOnPageCtr < itemsPerPage: if postsOnPageCtr < itemsPerPage:
if not headerOnly: if not headerOnly:
outboxItems['orderedItems'].append(p) boxItems['orderedItems'].append(p)
elif postsOnPageCtr == itemsPerPage: elif postsOnPageCtr == itemsPerPage:
# if this is the last post update the next message ID # if this is the last post update the next message ID
if '/statuses/' in p['id']: if '/statuses/' in p['id']:
postId = p['id'].split('/statuses/')[1].replace('/activity','') postId = p['id'].split('/statuses/')[1].replace('/activity','')
outboxHeader['next']= \ boxHeader['next']= \
httpPrefix+'://'+domain+'/users/'+ \ httpPrefix+'://'+domain+'/users/'+ \
nickname+'/outbox?max_id='+ \ nickname+'/'+boxname+'?max_id='+ \
postId+'&page=true' postId+'&page=true'
postsOnPageCtr += 1 postsOnPageCtr += 1
# remember the last post filename for use with prev # remember the last post filename for use with prev
@ -591,29 +607,31 @@ def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: st
except Exception as e: except Exception as e:
print(e) print(e)
if headerOnly: if headerOnly:
return outboxHeader return boxHeader
return outboxItems return boxItems
def archivePosts(nickname: str,domain: str,baseDir: str, \ def archivePosts(nickname: str,domain: str,baseDir: str, \
maxPostsInOutbox=256) -> None: boxname: str,maxPostsInBox=256) -> None:
"""Retain a maximum number of posts within the outbox """Retain a maximum number of posts within the given box
Move any others to an archive directory Move any others to an archive directory
""" """
outboxDir = createOutboxDir(nickname,domain,baseDir) if boxname!='inbox' and boxname!='outbox':
archiveDir = createOutboxArchive(nickname,domain,baseDir) return
postsInOutbox=sorted(os.listdir(outboxDir), reverse=False) boxDir = createPersonDir(nickname,domain,baseDir,boxname)
noOfPosts=len(postsInOutbox) archiveDir = createBoxArchive(nickname,domain,baseDir,boxname)
if noOfPosts<=maxPostsInOutbox: postsInBox=sorted(os.listdir(boxDir), reverse=False)
noOfPosts=len(postsInBox)
if noOfPosts<=maxPostsInBox:
return return
for postFilename in postsInOutbox: for postFilename in postsInBox:
filePath = os.path.join(outboxDir, postFilename) filePath = os.path.join(boxDir, postFilename)
if os.path.isfile(filePath): if os.path.isfile(filePath):
archivePath = os.path.join(archiveDir, postFilename) archivePath = os.path.join(archiveDir, postFilename)
os.rename(filePath,archivePath) os.rename(filePath,archivePath)
# TODO: possibly archive any associated media files # TODO: possibly archive any associated media files
noOfPosts -= 1 noOfPosts -= 1
if noOfPosts <= maxPostsInOutbox: if noOfPosts <= maxPostsInBox:
break break
def getPublicPostsOfPerson(nickname,domain,raw,simple): def getPublicPostsOfPerson(nickname,domain,raw,simple):

12
tests.py
View File

@ -115,7 +115,8 @@ def createServerAlice(path: str,domain: str,port: int,federationList: []):
useTor=False useTor=False
clientToServer=False clientToServer=False
privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,nickname,domain,port,httpPrefix,True) privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,nickname,domain,port,httpPrefix,True)
deleteAllPosts(path,nickname,domain) deleteAllPosts(path,nickname,domain,'inbox')
deleteAllPosts(path,nickname,domain,'outbox')
followPerson(path,nickname,domain,'bob','127.0.0.100:61936',federationList) followPerson(path,nickname,domain,'bob','127.0.0.100:61936',federationList)
followerOfPerson(path,nickname,domain,'bob','127.0.0.100:61936',federationList) followerOfPerson(path,nickname,domain,'bob','127.0.0.100:61936',federationList)
createPublicPost(path,nickname, domain, port,httpPrefix, "No wise fish would go anywhere without a porpoise", False, True, clientToServer) createPublicPost(path,nickname, domain, port,httpPrefix, "No wise fish would go anywhere without a porpoise", False, True, clientToServer)
@ -137,7 +138,8 @@ def createServerBob(path: str,domain: str,port: int,federationList: []):
useTor=False useTor=False
clientToServer=False clientToServer=False
privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,nickname,domain,port,httpPrefix,True) privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,nickname,domain,port,httpPrefix,True)
deleteAllPosts(path,nickname,domain) deleteAllPosts(path,nickname,domain,'inbox')
deleteAllPosts(path,nickname,domain,'outbox')
followPerson(path,nickname,domain,'alice','127.0.0.50:61935',federationList) followPerson(path,nickname,domain,'alice','127.0.0.50:61935',federationList)
followerOfPerson(path,nickname,domain,'alice','127.0.0.50:61935',federationList) followerOfPerson(path,nickname,domain,'alice','127.0.0.50:61935',federationList)
createPublicPost(path,nickname, domain, port,httpPrefix, "It's your life, live it your way.", False, True, clientToServer) createPublicPost(path,nickname, domain, port,httpPrefix, "It's your life, live it your way.", False, True, clientToServer)
@ -289,10 +291,12 @@ def testCreatePerson():
os.chdir(baseDir) os.chdir(baseDir)
privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(baseDir,nickname,domain,port,httpPrefix,True) privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(baseDir,nickname,domain,port,httpPrefix,True)
deleteAllPosts(baseDir,nickname,domain) deleteAllPosts(baseDir,nickname,domain,'inbox')
deleteAllPosts(baseDir,nickname,domain,'outbox')
setPreferredNickname(baseDir,nickname,domain,'badger') setPreferredNickname(baseDir,nickname,domain,'badger')
setBio(baseDir,nickname,domain,'Randomly roaming in your backyard') setBio(baseDir,nickname,domain,'Randomly roaming in your backyard')
archivePosts(nickname,domain,baseDir,4) archivePosts(nickname,domain,baseDir,'inbox',4)
archivePosts(nickname,domain,baseDir,'outbox',4)
createPublicPost(baseDir,nickname, domain, port,httpPrefix, "G'day world!", False, True, clientToServer, None, None, 'Not suitable for Vogons') createPublicPost(baseDir,nickname, domain, port,httpPrefix, "G'day world!", False, True, clientToServer, None, None, 'Not suitable for Vogons')
os.chdir(currDir) os.chdir(currDir)