From db68b34cc59bab3af973c6d3ff79f1d2be7bde61 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 8 Jul 2019 14:30:04 +0100 Subject: [PATCH] Capability on post is a list --- README.md | 8 ++++- daemon.py | 2 +- epicyon.py | 6 ++-- follow.py | 2 +- inbox.py | 102 ++++++++++++++++++++++++++++++----------------------- person.py | 2 +- posts.py | 84 ++++++++++++++++++++++++++++++++++++------- 7 files changed, 143 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index fab16cd5..6dd1dbc9 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Follow Accept from **Bob** to **Alice** with attached capabilities. 'type': 'Accept'} ``` -When posts are subsequently sent from the following instance (server-to-server) they should have the corresponding capability id string attached within the Create wrapper. In the above example that would be **http://bobdomain.net/caps/rOYtHApyr4ZWDUgEE1KqjhTe0kI3T2wJ**. It should contain a random string which is hard to guess by brute force methods. +When posts are subsequently sent from the following instance (server-to-server) they should have the corresponding capability id string attached within the Create wrapper. To handle the *shared inbox* scenario this should be a list rather than a single string. In the above example that would be *['http://bobdomain.net/caps/rOYtHApyr4ZWDUgEE1KqjhTe0kI3T2wJ']*. It should contain a random string which is hard to guess by brute force methods. ``` text Alice @@ -141,6 +141,12 @@ Subsequently **Bob** could change the stored capabilities for **Alice** in their Object capabilities can be strictly enforced by adding the **--ocap** option when running the server. The only activities which it is not enforced upon are **Follow** and **Accept**. Anyone can create a follow request or accept updated capabilities. +## Object capabilities in the shared inbox scenario + +Shared inboxes are obviously essential for any kind of scalability, otherwise there would be vast amounts of duplicated messages being dumped onto the intertubes like a big truck. + +With the shared inbox instead of sending from Alice to 500 of her fans on a different instance - repeatedly sending the same message to individual inboxes - a single message is sent to its shared inbox (which has its own special account called 'inbox') and it then decides how to distribute that. If a list of capability ids is attached to the message which gets sent to the shared inbox then the receiving server can use that. + ## Some capabilities *inbox:write* - follower can post anything to your inbox diff --git a/daemon.py b/daemon.py index 16a7bb51..ec0fe8d6 100644 --- a/daemon.py +++ b/daemon.py @@ -491,7 +491,7 @@ class PubServer(BaseHTTPRequestHandler): else: if self.path == '/sharedInbox' or self.path == '/inbox': print('DEBUG: POST to shared inbox') - if self._updateInboxQueue('sharedinbox',messageJson): + if self._updateInboxQueue('inbox',messageJson): return self.send_response(200) self.end_headers() diff --git a/epicyon.py b/epicyon.py index c930faa9..69482f8a 100644 --- a/epicyon.py +++ b/epicyon.py @@ -332,9 +332,9 @@ if not os.path.isdir(baseDir+'/accounts/'+nickname+'@'+domain): setConfigParam(baseDir,'adminPassword',adminPassword) createPerson(baseDir,nickname,domain,port,httpPrefix,True,adminPassword) -if not os.path.isdir(baseDir+'/accounts/sharedinbox@'+domain): - print('Creating shared inbox') - createSharedInbox(baseDir,'sharedinbox',domain,port,httpPrefix) +if not os.path.isdir(baseDir+'/accounts/inbox@'+domain): + print('Creating shared inbox: inbox@'+domain) + createSharedInbox(baseDir,'inbox',domain,port,httpPrefix) if not os.path.isdir(baseDir+'/accounts/capabilities@'+domain): print('Creating capabilities account which can sign requests') diff --git a/follow.py b/follow.py index a8c03769..6f852c36 100644 --- a/follow.py +++ b/follow.py @@ -33,7 +33,7 @@ def getFollowersOfPerson(baseDir: str, \ for subdir, dirs, files in os.walk(baseDir+'/accounts'): for account in dirs: filename = os.path.join(subdir, account)+'/'+followFile - if account == handle or account.startswith('sharedinbox@'): + if account == handle or account.startswith('inbox@'): continue if not os.path.isfile(filename): continue diff --git a/inbox.py b/inbox.py index a963d59b..38e3f252 100644 --- a/inbox.py +++ b/inbox.py @@ -124,7 +124,7 @@ def savePostToInboxQueue(baseDir: str,httpPrefix: str,nickname: str, domain: str filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json' sharedInboxItem=False - if nickname=='sharedinbox': + if nickname=='inbox': sharedInboxItem=True newQueueItem = { @@ -179,56 +179,70 @@ def runInboxQueue(baseDir: str,httpPrefix: str,sendThreads: [],postLog: [],cache with open(queueFilename, 'r') as fp: queueJson=commentjson.load(fp) - # check that capabilities are accepted - capabilitiesPassed=False - if queueJson['post'].get('capability'): - if isinstance(queueJson['post']['capability'], dict): - if debug: - print('DEBUG: capability is a dictionary when it should be a string') - os.remove(queueFilename) - queue.pop(0) - continue - ocapFilename= \ - getOcapFilename(baseDir, \ - queueJson['nickname'],queueJson['domain'], \ - queueJson['post']['actor'],'accept') - if not os.path.isfile(ocapFilename): - if debug: - print('DEBUG: capabilities for '+ \ - queueJson['post']['actor']+' do not exist') - os.remove(queueFilename) - queue.pop(0) - continue - with open(ocapFilename, 'r') as fp: - oc=commentjson.load(fp) - if not oc.get('id'): + sentToSharedInbox=False + if queueJson['post'].get('actor'): + if queueJson['post']['actor'].endswith('/inbox'): + sentToSharedInbox=True + + if sentToSharedInbox: + # if this is arriving at the shared inbox then + # don't do the capabilities checks + capabilitiesPassed=True + # TODO how to handle capabilities in the shared inbox scenario? + # should 'capability' be a list instead of a single value? + else: + # check that capabilities are accepted + capabilitiesPassed=False + if queueJson['post'].get('capability'): + if not isinstance(queueJson['post']['capability'], list): if debug: - print('DEBUG: capabilities for '+queueJson['post']['actor']+' do not contain an id') + print('DEBUG: capability on post should be a list') os.remove(queueFilename) queue.pop(0) continue - if oc['id']!=queueJson['post']['capability']: + capabilityIdList=queueJson['post']['capability'] + + ocapFilename= \ + getOcapFilename(baseDir, \ + queueJson['nickname'],queueJson['domain'], \ + queueJson['post']['actor'],'accept') + if not os.path.isfile(ocapFilename): if debug: - print('DEBUG: capability id mismatch') + print('DEBUG: capabilities for '+ \ + queueJson['post']['actor']+' do not exist') os.remove(queueFilename) queue.pop(0) - continue - if not oc.get('capability'): + continue + with open(ocapFilename, 'r') as fp: + oc=commentjson.load(fp) + if not oc.get('id'): + if debug: + print('DEBUG: capabilities for '+queueJson['post']['actor']+' do not contain an id') + os.remove(queueFilename) + queue.pop(0) + continue + if oc['id']!=capabilityIdList[0]: + if debug: + print('DEBUG: capability id mismatch') + os.remove(queueFilename) + queue.pop(0) + continue + if not oc.get('capability'): + if debug: + print('DEBUG: missing capability list') + os.remove(queueFilename) + queue.pop(0) + continue + if not CapablePost(queueJson['post'],oc['capability'],debug): + if debug: + print('DEBUG: insufficient capabilities to write to inbox from '+ \ + queueJson['post']['actor']) + os.remove(queueFilename) + queue.pop(0) + continue if debug: - print('DEBUG: missing capability list') - os.remove(queueFilename) - queue.pop(0) - continue - if not CapablePost(queueJson['post'],oc['capability'],debug): - if debug: - print('DEBUG: insufficient capabilities to write to inbox from '+ \ - queueJson['post']['actor']) - os.remove(queueFilename) - queue.pop(0) - continue - if debug: - print('DEBUG: object capabilities check success') - capabilitiesPassed=True + print('DEBUG: object capabilities check success') + capabilitiesPassed=True if ocapAlways and not capabilitiesPassed: # Allow follow types through @@ -347,7 +361,7 @@ def runInboxQueue(baseDir: str,httpPrefix: str,sendThreads: [],postLog: [],cache # post already exists in this person's inbox continue # We could do this in a more storage space efficient way - # by linking to the inbox of sharedinbox@domain + # by linking to the inbox of inbox@domain # However, this allows for easy deletion by individuals # without affecting any other people copyfile(queueFilename, destination) diff --git a/person.py b/person.py index ba8066c8..a23d7cff 100644 --- a/person.py +++ b/person.py @@ -140,7 +140,7 @@ def validNickname(nickname: str) -> bool: for c in forbiddenChars: if c in nickname: return False - reservedNames=['inbox','outbox','following','followers','sharedinbox','capabilities'] + reservedNames=['inbox','outbox','following','followers','capabilities'] if nickname in reservedNames: return False return True diff --git a/posts.py b/posts.py index e19c87e8..b5a19ddd 100644 --- a/posts.py +++ b/posts.py @@ -351,19 +351,17 @@ def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \ # if capabilities have been granted for this actor # then get the corresponding id capabilityId=None - ocapFilename= getOcapFilename(baseDir,nickname,domain,toUrl,'granted') - #print('ocapFilename: '+ocapFilename) + capabilityIdList=[] + ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted') if os.path.isfile(ocapFilename): with open(ocapFilename, 'r') as fp: oc=commentjson.load(fp) if oc.get('id'): - capabilityId=oc['id'] - #else: - # print('ocapFilename: '+ocapFilename+' not found') + capabilityIdList=[oc['id']] newPost = { 'id': newPostId+'/activity', - 'capability': capabilityId, + 'capability': capabilityIdList, 'type': 'Create', 'actor': actorUrl, 'published': published, @@ -464,6 +462,32 @@ def outboxMessageCreateWrap(httpPrefix: str,nickname: str,domain: str, \ httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber return newPost +def postIsAddressedToFollowers(baseDir: str, + nickname: str, domain: str, port: int,httpPrefix: str, + postJson: {}) -> bool: + """Returns true if the given post is addressed to followers of the nickname + """ + if port!=80 and port!=443: + domain=domain+':'+str(port) + + if not postJson.get('object'): + return False + if not postJson['object'].get('to'): + return False + + followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers' + + # does the followers url exist in 'to' or 'cc' lists? + addressedToFollowers=False + if followersUrl in postJson['object']['to']: + addressedToFollowers=True + if not addressedToFollowers: + if not postJson['object'].get('cc'): + return False + if followersUrl in postJson['object']['cc']: + addressedToFollowers=True + return addressedToFollowers + def createPublicPost(baseDir: str, nickname: str, domain: str, port: int,httpPrefix: str, \ content: str, followersOnly: bool, saveToFile: bool, @@ -626,6 +650,10 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ """ withDigest=True + sharedInbox=False + if toNickname=='inbox': + sharedInbox=True + if toPort!=80 and toPort!=443: if ':' not in toDomain: toDomain=toDomain+':'+str(toPort) @@ -633,7 +661,7 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ handle=httpPrefix+'://'+toDomain+'/@'+toNickname # lookup the inbox for the To handle - wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers) + wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers) if not wfRequest: if debug: print('DEBUG: webfinger for '+handle+' failed') @@ -645,18 +673,16 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ postToBox='outbox' # get the actor inbox/outbox/capabilities for the To handle - inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition = \ + inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition = \ getPersonBox(session,wfRequest,personCache,postToBox) - # If there are more than one followers on the target domain - # then send to teh shared inbox indead of the individual inbox if nickname=='capabilities': inboxUrl=capabilityAcquisition if not capabilityAcquisition: return 2 else: - if noOfFollowersOnDomain(baseDir,handle,toDomain)>1 and sharedInbox: - inboxUrl=sharedInbox + if sharedInbox and sharedInboxUrl: + inboxUrl=sharedInboxUrl if debug: print('DEBUG: Sending to endpoint '+inboxUrl) @@ -702,6 +728,40 @@ def sendSignedJson(postJsonObject: {},session,baseDir: str, \ thr.start() return 0 +def sendToFollowers(session,baseDir: str, + nickname: str, domain: str, port: int,httpPrefix: str, + postJsonObject: {}): + """sends a post to the followers of the given nickname + """ + if not postIsAddressedToFollowers(baseDir,nickname,domain, \ + port,httpPrefix,postJsonObject): + return + + grouped=groupFollowersByDomain(baseDir,nickname,domain) + if not grouped: + return + + # for each instance + for followerDomain,followerHandles in grouped.items(): + toPort=port + index=0 + toDomain=followerHandles[index].split('@')[1] + if ':' in toDomain: + toPort=toDomain.split(':')[1] + toDomain=toDomain.split(':')[0] + toNickname=followerHandles[index].split('@')[0] + cc='' + if len(followerHandles)>1: + nickname='inbox' + toNickname='inbox' + sendSignedJson(postJsonObject,session,baseDir, \ + nickname,domain,port, \ + toNickname,toDomain,toPort, \ + cc,httpPrefix,True,clientToServer, \ + federationList,ocapGranted, \ + sendThreads,postLog,cachedWebfingers, \ + personCache,debug) + def createInbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \ itemsPerPage: int,headerOnly: bool,pageNumber=None) -> {}: return createBoxBase(baseDir,'inbox',nickname,domain,port,httpPrefix, \