epicyon/inbox.py

1262 lines
51 KiB
Python
Raw Normal View History

2019-06-28 21:59:54 +00:00
__filename__ = "inbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "0.0.1"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
import json
import os
2019-06-29 10:08:59 +00:00
import datetime
2019-07-04 12:23:53 +00:00
import time
import json
import commentjson
from shutil import copyfile
2019-07-02 10:39:55 +00:00
from utils import urlPermitted
2019-07-04 10:02:56 +00:00
from utils import createInboxQueueDir
2019-07-06 13:49:25 +00:00
from utils import getStatusNumber
2019-07-09 14:20:23 +00:00
from utils import getDomainFromActor
from utils import getNicknameFromActor
from utils import domainPermitted
2019-07-11 12:29:31 +00:00
from utils import locatePost
2019-07-14 16:37:01 +00:00
from utils import deletePost
2019-07-14 16:57:06 +00:00
from utils import removeAttachment
2019-07-04 12:23:53 +00:00
from httpsig import verifyPostHeaders
from session import createSession
2019-07-04 19:34:28 +00:00
from session import getJson
2019-07-04 12:23:53 +00:00
from follow import receiveFollowRequest
2019-07-08 18:55:39 +00:00
from follow import getFollowersOfActor
2019-07-17 11:54:13 +00:00
from follow import unfollowerOfPerson
2019-07-04 14:36:29 +00:00
from pprint import pprint
2019-07-04 19:34:28 +00:00
from cache import getPersonFromCache
2019-07-04 20:25:19 +00:00
from cache import storePersonInCache
2019-07-06 15:17:21 +00:00
from acceptreject import receiveAcceptReject
2019-07-07 15:51:04 +00:00
from capabilities import getOcapFilename
2019-07-07 22:06:46 +00:00
from capabilities import CapablePost
2019-07-09 14:20:23 +00:00
from capabilities import capabilitiesReceiveUpdate
2019-07-10 12:40:31 +00:00
from like import updateLikesCollection
2019-07-12 09:10:09 +00:00
from like import undoLikesCollectionEntry
from blocking import isBlocked
2019-07-14 20:50:27 +00:00
from filters import isFiltered
from announce import updateAnnounceCollection
2019-07-04 19:34:28 +00:00
def validInbox(baseDir: str,nickname: str,domain: str) -> bool:
2019-07-18 11:35:48 +00:00
"""Checks whether files were correctly saved to the inbox
"""
if ':' in domain:
domain=domain.split(':')[0]
inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
if not os.path.isdir(inboxDir):
return True
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
filename = os.path.join(subdir, f)
if not os.path.isfile(filename):
print('filename: '+filename)
return False
if 'postNickname' in open(filename).read():
2019-07-18 11:35:48 +00:00
print('queue file incorrectly saved to '+filename)
return False
return True
def validInboxFilenames(baseDir: str,nickname: str,domain: str, \
expectedDomain: str,expectedPort: int) -> bool:
"""Used by unit tests to check that the port number gets appended to
domain names within saved post filenames
"""
if ':' in domain:
domain=domain.split(':')[0]
inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
if not os.path.isdir(inboxDir):
return True
expectedStr=expectedDomain+':'+str(expectedPort)
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
filename = os.path.join(subdir, f)
if not os.path.isfile(filename):
print('filename: '+filename)
return False
if not expectedStr in filename:
print('Invalid filename: '+filename)
return False
return True
2019-07-04 19:34:28 +00:00
def getPersonPubKey(session,personUrl: str,personCache: {},debug: bool) -> str:
if not personUrl:
return None
personUrl=personUrl.replace('#main-key','')
personJson = getPersonFromCache(personUrl,personCache)
if not personJson:
if debug:
print('DEBUG: Obtaining public key for '+personUrl)
2019-07-04 20:25:19 +00:00
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
2019-07-04 19:34:28 +00:00
personJson = getJson(session,personUrl,asHeader,None)
if not personJson:
return None
pubKey=None
if personJson.get('publicKey'):
if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem']
else:
if personJson.get('publicKeyPem'):
pubKey=personJson['publicKeyPem']
if not pubKey:
if debug:
print('DEBUG: Public key not found for '+personUrl)
storePersonInCache(personUrl,personJson,personCache)
return pubKey
2019-06-28 21:59:54 +00:00
2019-07-02 15:07:27 +00:00
def inboxMessageHasParams(messageJson: {}) -> bool:
"""Checks whether an incoming message contains expected parameters
"""
2019-07-06 13:49:25 +00:00
expectedParams=['type','actor','object']
2019-07-02 15:07:27 +00:00
for param in expectedParams:
if not messageJson.get(param):
return False
2019-07-06 13:49:25 +00:00
if not messageJson.get('to'):
2019-07-17 11:54:13 +00:00
allowedWithoutToParam=['Follow','Request','Capability','Undo']
2019-07-06 13:49:25 +00:00
if messageJson['type'] not in allowedWithoutToParam:
return False
2019-07-02 15:07:27 +00:00
return True
2019-07-09 14:20:23 +00:00
def inboxPermittedMessage(domain: str,messageJson: {},federationList: []) -> bool:
2019-06-28 21:59:54 +00:00
""" check that we are receiving from a permitted domain
"""
testParam='actor'
if not messageJson.get(testParam):
return False
actor=messageJson[testParam]
# always allow the local domain
2019-07-01 11:48:54 +00:00
if domain in actor:
2019-06-28 21:59:54 +00:00
return True
2019-07-09 14:20:23 +00:00
if not urlPermitted(actor,federationList,"inbox:write"):
2019-06-28 21:59:54 +00:00
return False
2019-07-11 17:55:10 +00:00
if messageJson['type']!='Follow' and \
messageJson['type']!='Like' and \
2019-07-11 21:38:28 +00:00
messageJson['type']!='Delete' and \
2019-07-11 17:55:10 +00:00
messageJson['type']!='Announce':
2019-07-06 13:49:25 +00:00
if messageJson.get('object'):
2019-07-15 09:20:16 +00:00
if not isinstance(messageJson['object'], dict):
return False
2019-07-06 13:49:25 +00:00
if messageJson['object'].get('inReplyTo'):
inReplyTo=messageJson['object']['inReplyTo']
2019-07-09 14:20:23 +00:00
if not urlPermitted(inReplyTo,federationList):
2019-07-06 13:49:25 +00:00
return False
2019-06-28 21:59:54 +00:00
return True
2019-06-29 10:08:59 +00:00
2019-07-02 20:54:22 +00:00
def validPublishedDate(published) -> bool:
2019-06-29 10:08:59 +00:00
currTime=datetime.datetime.utcnow()
pubDate=datetime.datetime.strptime(published,"%Y-%m-%dT%H:%M:%SZ")
daysSincePublished = (currTime - pubTime).days
if daysSincePublished>30:
return False
return True
2019-07-04 10:02:56 +00:00
2019-07-28 13:30:19 +00:00
def savePostToInboxQueue(baseDir: str,httpPrefix: str,nickname: str, domain: str,postJsonObject: {},host: str,headers: str,postPath: str,postFromWebInterface: bool,debug: bool) -> str:
2019-07-04 10:02:56 +00:00
"""Saves the give json to the inbox queue for the person
keyId specifies the actor sending the post
"""
2019-07-18 11:35:48 +00:00
originalDomain=domain
2019-07-04 10:02:56 +00:00
if ':' in domain:
domain=domain.split(':')[0]
# block at the ealiest stage possible, which means the data
# isn't written to file
2019-07-15 10:22:19 +00:00
postNickname=None
postDomain=None
if postJsonObject.get('actor'):
postNickname=getNicknameFromActor(postJsonObject['actor'])
postDomain,postPort=getDomainFromActor(postJsonObject['actor'])
if isBlocked(baseDir,nickname,domain,postNickname,postDomain):
return None
2019-07-15 10:22:19 +00:00
if postPort:
if postPort!=80 and postPort!=443:
postDomain=postDomain+':'+str(postPort)
2019-07-14 20:50:27 +00:00
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('content'):
if isinstance(postJsonObject['object']['content'], str):
if isFiltered(baseDir,nickname,domain,postJsonObject['object']['content']):
return None
2019-07-14 16:57:06 +00:00
if postJsonObject.get('id'):
postId=postJsonObject['id'].replace('/activity','')
2019-07-06 13:49:25 +00:00
else:
statusNumber,published = getStatusNumber()
2019-07-18 11:35:48 +00:00
postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
2019-07-06 13:49:25 +00:00
2019-07-04 10:09:27 +00:00
currTime=datetime.datetime.utcnow()
published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
2019-07-05 11:27:18 +00:00
inboxQueueDir=createInboxQueueDir(nickname,domain,baseDir)
handle=nickname+'@'+domain
destination=baseDir+'/accounts/'+handle+'/inbox/'+postId.replace('/','#')+'.json'
if os.path.isfile(destination):
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: inbox item already exists')
return None
filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json'
sharedInboxItem=False
2019-07-08 13:30:04 +00:00
if nickname=='inbox':
sharedInboxItem=True
2019-07-04 14:36:29 +00:00
newQueueItem = {
2019-07-15 09:20:16 +00:00
'id': postId,
2019-07-07 15:51:04 +00:00
'nickname': nickname,
'domain': domain,
2019-07-15 10:22:19 +00:00
'postNickname': postNickname,
'postDomain': postDomain,
'sharedInbox': sharedInboxItem,
2019-07-04 10:09:27 +00:00
'published': published,
2019-07-04 20:25:19 +00:00
'host': host,
2019-07-04 12:23:53 +00:00
'headers': headers,
2019-07-05 22:13:20 +00:00
'path': postPath,
2019-07-14 16:57:06 +00:00
'post': postJsonObject,
'filename': filename,
2019-07-28 13:30:19 +00:00
'destination': destination,
'postFromWebInterface': postFromWebInterface
2019-07-04 10:02:56 +00:00
}
2019-07-06 13:49:25 +00:00
if debug:
print('Inbox queue item created')
pprint(newQueueItem)
2019-07-04 10:02:56 +00:00
with open(filename, 'w') as fp:
commentjson.dump(newQueueItem, fp, indent=4, sort_keys=False)
return filename
2019-07-04 12:23:53 +00:00
2019-07-08 18:55:39 +00:00
def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \
actor: str,queue: [],queueJson: {}, \
capabilityId: str,debug : bool) -> bool:
if nickname=='inbox':
return True
ocapFilename= \
getOcapFilename(baseDir, \
queueJson['nickname'],queueJson['domain'], \
actor,'accept')
if not os.path.isfile(ocapFilename):
if debug:
print('DEBUG: capabilities for '+ \
actor+' do not exist')
os.remove(queueFilename)
queue.pop(0)
return False
with open(ocapFilename, 'r') as fp:
oc=commentjson.load(fp)
if not oc.get('id'):
if debug:
print('DEBUG: capabilities for '+actor+' do not contain an id')
os.remove(queueFilename)
queue.pop(0)
return False
if oc['id']!=capabilityId:
if debug:
print('DEBUG: capability id mismatch')
os.remove(queueFilename)
queue.pop(0)
return False
if not oc.get('capability'):
if debug:
print('DEBUG: missing capability list')
os.remove(queueFilename)
queue.pop(0)
return False
if not CapablePost(queueJson['post'],oc['capability'],debug):
if debug:
print('DEBUG: insufficient capabilities to write to inbox from '+actor)
os.remove(queueFilename)
queue.pop(0)
return False
if debug:
print('DEBUG: object capabilities check success')
return True
2019-07-08 22:12:24 +00:00
def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \
recipientsDict :{}, \
domainMatch: str,domain :str, \
2019-07-11 12:29:31 +00:00
actor :str,debug: bool) -> bool:
2019-07-08 22:12:24 +00:00
"""Given a list of post recipients (toList) from 'to' or 'cc' parameters
populate a recipientsDict with the handle and capabilities id for each
"""
followerRecipients=False
for recipient in toList:
# is this a to a local account?
if domainMatch in recipient:
# get the handle for the local account
nickname=recipient.split(domainMatch)[1]
handle=nickname+'@'+domain
if os.path.isdir(baseDir+'/accounts/'+handle):
# are capabilities granted for this account to the
# sender (actor) of the post?
ocapFilename=baseDir+'/accounts/'+handle+'/ocap/accept/'+actor.replace('/','#')+'.json'
if os.path.isfile(ocapFilename):
# read the granted capabilities and obtain the id
with open(ocapFilename, 'r') as fp:
ocapJson=commentjson.load(fp)
if ocapJson.get('id'):
# append with the capabilities id
recipientsDict[handle]=ocapJson['id']
else:
recipientsDict[handle]=None
else:
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: '+ocapFilename+' not found')
2019-07-08 22:12:24 +00:00
recipientsDict[handle]=None
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: '+baseDir+'/accounts/'+handle+' does not exist')
else:
if debug:
print('DEBUG: '+recipient+' is not local to '+domainMatch)
print(str(toList))
2019-07-08 22:12:24 +00:00
if recipient.endswith('followers'):
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: followers detected as post recipients')
2019-07-08 22:12:24 +00:00
followerRecipients=True
return followerRecipients,recipientsDict
2019-07-11 12:29:31 +00:00
def inboxPostRecipients(baseDir :str,postJsonObject :{},httpPrefix :str,domain : str,port :int, debug :bool) -> ([],[]):
"""Returns dictionaries containing the recipients of the given post
The shared dictionary contains followers
"""
2019-07-08 22:12:24 +00:00
recipientsDict={}
recipientsDictFollowers={}
2019-07-08 22:12:24 +00:00
if not postJsonObject.get('actor'):
2019-07-11 12:29:31 +00:00
if debug:
pprint(postJsonObject)
print('WARNING: inbox post has no actor')
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
if ':' in domain:
domain=domain.split(':')[0]
domainBase=domain
if port!=80 and port!=443:
domain=domain+':'+str(port)
domainMatch='/'+domain+'/users/'
actor = postJsonObject['actor']
# first get any specific people which the post is addressed to
followerRecipients=False
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('to'):
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: resolving "to"')
2019-07-08 22:12:24 +00:00
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
postJsonObject['object']['to'], \
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no "to"')
2019-07-08 22:12:24 +00:00
if postJsonObject['object'].get('cc'):
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
postJsonObject['object']['cc'], \
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no cc')
else:
if debug:
if isinstance(postJsonObject['object'], str):
if '/statuses/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to a post')
else:
if '/users/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to an actor')
2019-07-08 22:12:24 +00:00
if postJsonObject.get('to'):
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
postJsonObject['to'], \
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
if postJsonObject.get('cc'):
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
postJsonObject['cc'], \
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
if not followerRecipients:
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: no followers were resolved')
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
# now resolve the followers
recipientsDictFollowers= \
2019-07-11 12:29:31 +00:00
getFollowersOfActor(baseDir,actor,debug)
2019-07-08 22:12:24 +00:00
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
2019-07-17 10:34:00 +00:00
def receiveUndoFollow(session,baseDir: str,httpPrefix: str, \
2019-07-17 10:38:10 +00:00
port: int,messageJson: {}, \
federationList: [], \
debug : bool) -> bool:
2019-07-17 10:34:00 +00:00
if not messageJson['object'].get('actor'):
if debug:
print('DEBUG: follow request has no actor within object')
return False
if '/users/' not in messageJson['object']['actor']:
if debug:
print('DEBUG: "users" missing from actor within object')
return False
if messageJson['object']['actor'] != messageJson['actor']:
if debug:
print('DEBUG: actors do not match')
return False
nicknameFollower=getNicknameFromActor(messageJson['object']['actor'])
domainFollower,portFollower=getDomainFromActor(messageJson['object']['actor'])
domainFollowerFull=domainFollower
if portFollower:
if portFollower!=80 and portFollower!=443:
domainFollowerFull=domainFollower+':'+str(portFollower)
nicknameFollowing=getNicknameFromActor(messageJson['object']['object'])
domainFollowing,portFollowing=getDomainFromActor(messageJson['object']['object'])
domainFollowingFull=domainFollowing
if portFollowing:
if portFollowing!=80 and portFollowing!=443:
domainFollowingFull=domainFollowing+':'+str(portFollowing)
2019-07-17 11:54:13 +00:00
if unfollowerOfPerson(baseDir, \
nicknameFollowing,domainFollowingFull, \
nicknameFollower,domainFollowerFull, \
debug):
if debug:
print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was removed')
return True
if debug:
print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was not removed')
return False
2019-07-17 10:34:00 +00:00
def receiveUndo(session,baseDir: str,httpPrefix: str, \
port: int,sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
messageJson: {},federationList: [], \
debug : bool, \
acceptedCaps=["inbox:write","objects:read"]) -> bool:
"""Receives an undo request within the POST section of HTTPServer
"""
if not messageJson['type'].startswith('Undo'):
return False
2019-07-17 11:24:11 +00:00
if debug:
print('DEBUG: Undo activity received')
2019-07-17 10:34:00 +00:00
if not messageJson.get('actor'):
if debug:
print('DEBUG: follow request has no actor')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor')
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], dict):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a dict')
return False
if not messageJson['object'].get('type'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object type')
return False
if not messageJson['object'].get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object within object')
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object within object is not a string')
return False
if messageJson['object']['type']=='Follow':
return receiveUndoFollow(session,baseDir,httpPrefix, \
2019-07-17 10:38:10 +00:00
port,messageJson, \
federationList, \
debug)
2019-07-17 10:34:00 +00:00
return False
2019-07-09 14:20:23 +00:00
def receiveUpdate(session,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an Update activity within the POST section of HTTPServer
"""
if messageJson['type']!='Update':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], dict):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a dict')
return False
if not messageJson['object'].get('type'):
if debug:
print('DEBUG: '+messageJson['type']+' object has no type')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type'])
return False
if messageJson['object'].get('capability') and messageJson['object'].get('scope'):
domain,tempPort=getDomainFromActor(messageJson['object']['scope'])
nickname=getNicknameFromActor(messageJson['object']['scope'])
if messageJson['object']['type']=='Capability':
if capabilitiesReceiveUpdate(baseDir,nickname,domain,port,
messageJson['actor'], \
messageJson['object']['id'], \
messageJson['object']['capability'], \
debug):
if debug:
print('DEBUG: An update was received')
return True
return False
2019-07-10 12:40:31 +00:00
def receiveLike(session,handle: str,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives a Like activity within the POST section of HTTPServer
"""
if messageJson['type']!='Like':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type'])
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of like - '+handle)
# if this post in the outbox of the person?
2019-07-11 12:29:31 +00:00
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
2019-07-10 12:40:31 +00:00
if not postFilename:
if debug:
print('DEBUG: post not found in inbox or outbox')
print(messageJson['object'])
return True
if debug:
2019-07-11 12:59:00 +00:00
print('DEBUG: liked post found in inbox')
updateLikesCollection(postFilename,messageJson['object'],messageJson['actor'],debug)
2019-07-10 12:40:31 +00:00
return True
2019-07-12 09:10:09 +00:00
def receiveUndoLike(session,handle: str,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an undo like activity within the POST section of HTTPServer
"""
if messageJson['type']!='Undo':
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('type'):
return False
if messageJson['object']['type']!='Like':
return False
if not messageJson['object'].get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' like has no object')
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' like object is not a string')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type']+' like')
return False
if '/statuses/' not in messageJson['object']['object']:
if debug:
print('DEBUG: "statuses" missing from like object in '+messageJson['type'])
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of undo like - '+handle)
# if this post in the outbox of the person?
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object']['object'])
if not postFilename:
if debug:
2019-07-12 09:41:57 +00:00
print('DEBUG: unliked post not found in inbox or outbox')
2019-07-12 09:10:09 +00:00
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: liked post found in inbox. Now undoing.')
undoLikesCollectionEntry(postFilename,messageJson['object'],messageJson['actor'],debug)
return True
2019-07-11 21:38:28 +00:00
def receiveDelete(session,handle: str,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives a Delete activity within the POST section of HTTPServer
"""
if messageJson['type']!='Delete':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
2019-07-17 17:16:48 +00:00
if debug:
print('DEBUG: Delete activity arrived')
2019-07-11 21:38:28 +00:00
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type'])
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
if messageJson['actor'] not in messageJson['object']:
if debug:
print('DEBUG: actor is not the owner of the post to be deleted')
2019-07-11 21:38:28 +00:00
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of like - '+handle)
# if this post in the outbox of the person?
2019-07-17 17:16:48 +00:00
messageId=messageJson['object'].replace('/activity','')
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageId)
2019-07-11 21:38:28 +00:00
if not postFilename:
if debug:
print('DEBUG: delete post not found in inbox or outbox')
2019-07-17 17:16:48 +00:00
print(messageId)
return True
2019-07-14 17:02:41 +00:00
deletePost(baseDir,httpPrefix,handle.split('@')[0],handle.split('@')[1],postFilename,debug)
2019-07-11 21:38:28 +00:00
if debug:
print('DEBUG: post deleted - '+postFilename)
return True
2019-07-11 19:31:02 +00:00
def receiveAnnounce(session,handle: str,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
2019-07-12 09:41:57 +00:00
"""Receives an announce activity within the POST section of HTTPServer
2019-07-11 19:31:02 +00:00
"""
if messageJson['type']!='Announce':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: receiving announce on '+handle)
2019-07-11 19:31:02 +00:00
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type'])
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of announce - '+handle)
# is this post in the outbox of the person?
2019-07-11 19:31:02 +00:00
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
if not postFilename:
if debug:
print('DEBUG: announce post not found in inbox or outbox')
print(messageJson['object'])
return True
updateAnnounceCollection(postFilename,messageJson['actor'],debug)
2019-07-11 19:31:02 +00:00
if debug:
print('DEBUG: announced/repeated post found in inbox')
return True
2019-07-12 09:41:57 +00:00
def receiveUndoAnnounce(session,handle: str,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an undo announce activity within the POST section of HTTPServer
"""
if messageJson['type']!='Undo':
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('object'):
return False
if not isinstance(messageJson['object']['object'], str):
return False
if messageJson['object']['type']!='Announce':
return False
if '/users/' not in messageJson['actor']:
if debug:
print('DEBUG: "users" missing from actor in '+messageJson['type']+' announce')
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type']+' announce')
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of undo announce - '+handle)
# if this post in the outbox of the person?
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
if not postFilename:
if debug:
print('DEBUG: undo announce post not found in inbox or outbox')
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: announced/repeated post to be undone found in inbox')
with open(postFilename, 'r') as fp:
2019-07-14 16:57:06 +00:00
postJsonObject=commentjson.load(fp)
if not postJsonObject.get('type'):
if postJsonObject['type']!='Announce':
if debug:
print("DEBUG: Attempt to undo something which isn't an announcement")
return False
undoAnnounceCollectionEntry(postFilename,messageJson['actor'],debug)
2019-07-12 09:41:57 +00:00
os.remove(postFilename)
return True
def populateReplies(baseDir :str,httpPrefix :str,domain :str, \
2019-07-13 21:00:12 +00:00
messageJson :{},maxReplies: int,debug :bool) -> bool:
"""Updates the list of replies for a post on this domain if
a reply to it arrives
"""
if not messageJson.get('id'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('inReplyTo'):
return False
if not messageJson['object'].get('to'):
return False
replyTo=messageJson['object']['inReplyTo']
if debug:
print('DEBUG: post contains a reply')
# is this a reply to a post on this domain?
if not replyTo.startswith(httpPrefix+'://'+domain+'/'):
if debug:
print('DEBUG: post is a reply to another not on this domain')
return False
replyToNickname=getNicknameFromActor(replyTo)
if not replyToNickname:
if debug:
print('DEBUG: no nickname found for '+replyTo)
return False
replyToDomain,replyToPort=getDomainFromActor(replyTo)
if not replyToDomain:
if debug:
print('DEBUG: no domain found for '+replyTo)
return False
postFilename=locatePost(baseDir,replyToNickname,replyToDomain,replyTo)
if not postFilename:
if debug:
print('DEBUG: post may have expired - '+replyTo)
2019-07-13 19:28:14 +00:00
return False
# populate a text file containing the ids of replies
postRepliesFilename=postFilename.replace('.json','.replies')
messageId=messageJson['id'].replace('/activity','')
if os.path.isfile(postRepliesFilename):
2019-07-13 21:00:12 +00:00
numLines = sum(1 for line in open(postRepliesFilename))
if numlines>maxReplies:
return False
2019-07-13 19:28:14 +00:00
if messageId not in open(postRepliesFilename).read():
repliesFile=open(postRepliesFilename, "a")
repliesFile.write(messageId+'\n')
repliesFile.close()
else:
repliesFile=open(postRepliesFilename, "w")
repliesFile.write(messageId+'\n')
repliesFile.close()
return True
2019-07-10 12:40:31 +00:00
def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
baseDir: str,httpPrefix: str,sendThreads: [], \
postLog: [],cachedWebfingers: {},personCache: {}, \
queue: [],domain: str,port: int,useTor: bool, \
federationList: [],ocapAlways: bool,debug: bool, \
2019-07-10 13:32:47 +00:00
acceptedCaps: [],
2019-07-13 21:00:12 +00:00
queueFilename :str,destinationFilename :str,
maxReplies: int,allowDeletion: bool) -> bool:
""" Anything which needs to be done after capabilities checks have passed
"""
2019-07-10 12:40:31 +00:00
if receiveLike(session,handle, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
print('DEBUG: Like accepted from '+keyId)
return False
2019-07-12 09:10:09 +00:00
if receiveUndoLike(session,handle, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
print('DEBUG: Undo like accepted from '+keyId)
return False
2019-07-11 19:31:02 +00:00
if receiveAnnounce(session,handle, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
print('DEBUG: Announce accepted from '+keyId)
2019-07-12 09:41:57 +00:00
if receiveUndoAnnounce(session,handle, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
print('DEBUG: Undo announce accepted from '+keyId)
2019-07-12 11:35:03 +00:00
return False
2019-07-12 09:41:57 +00:00
if allowDeletion:
2019-07-17 17:44:26 +00:00
if receiveDelete(session,handle, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
print('DEBUG: Delete accepted from '+keyId)
return False
2019-07-13 21:00:12 +00:00
populateReplies(baseDir,httpPrefix,domain,messageJson,maxReplies,debug)
2019-07-10 13:32:47 +00:00
if debug:
print('DEBUG: object capabilities passed')
print('copy from '+queueFilename+' to '+destinationFilename)
if messageJson.get('postNickname'):
with open(destinationFilename, 'w') as fp:
commentjson.dump(messageJson['post'], fp, indent=4, sort_keys=False)
else:
with open(destinationFilename, 'w') as fp:
commentjson.dump(messageJson, fp, indent=4, sort_keys=False)
return True
2019-07-12 21:09:23 +00:00
def restoreQueueItems(baseDir: str,queue: []) -> None:
"""Checks the queue for each account and appends filenames
"""
queue=[]
for subdir,dirs,files in os.walk(baseDir+'/accounts'):
for account in dirs:
queueDir=baseDir+'/accounts/'+account+'/queue'
if os.path.isdir(queueDir):
for queuesubdir,queuedirs,queuefiles in os.walk(queueDir):
for qfile in queuefiles:
queue.append(os.path.join(queueDir, qfile))
2019-07-11 21:38:28 +00:00
def runInboxQueue(baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {},queue: [], \
domain: str,port: int,useTor: bool,federationList: [], \
2019-07-15 10:22:19 +00:00
ocapAlways: bool,maxReplies: int, \
domainMaxPostsPerDay: int,accountMaxPostsPerDay: int, \
allowDeletion: bool,debug: bool, \
2019-07-11 21:38:28 +00:00
acceptedCaps=["inbox:write","objects:read"]) -> None:
2019-07-04 12:23:53 +00:00
"""Processes received items and moves them to
the appropriate directories
"""
currSessionTime=int(time.time())
sessionLastUpdate=currSessionTime
session=createSession(domain,port,useTor)
2019-07-08 23:05:48 +00:00
inboxHandle='inbox@'+domain
2019-07-04 12:23:53 +00:00
if debug:
print('DEBUG: Inbox queue running')
2019-07-12 21:09:23 +00:00
# if queue processing was interrupted (eg server crash)
# then this loads any outstanding items back into the queue
restoreQueueItems(baseDir,queue)
2019-07-15 10:22:19 +00:00
# keep track of numbers of incoming posts per unit of time
quotasLastUpdate=int(time.time())
quotas={
'domains': {},
'accounts': {}
}
2019-07-04 12:23:53 +00:00
while True:
2019-07-07 22:58:12 +00:00
time.sleep(1)
2019-07-04 12:23:53 +00:00
if len(queue)>0:
2019-07-15 10:22:19 +00:00
currTime=int(time.time())
# recreate the session periodically
if currTime-sessionLastUpdate>1200:
2019-07-04 12:23:53 +00:00
session=createSession(domain,port,useTor)
2019-07-15 10:22:19 +00:00
sessionLastUpdate=currTime
2019-07-04 12:23:53 +00:00
# oldest item first
queue.sort()
queueFilename=queue[0]
if not os.path.isfile(queueFilename):
if debug:
print("DEBUG: queue item rejected becase it has no file: "+queueFilename)
queue.pop(0)
continue
# Load the queue json
with open(queueFilename, 'r') as fp:
queueJson=commentjson.load(fp)
2019-07-15 10:22:19 +00:00
# clear the daily quotas for maximum numbers of received posts
if currTime-quotasLastUpdate>60*60*24:
quotas={
'domains': {},
'accounts': {}
}
quotasLastUpdate=currTime
# limit the number of posts which can arrive per domain per day
postDomain=queueJson['postDomain']
if postDomain:
2019-07-15 10:25:13 +00:00
if domainMaxPostsPerDay>0:
if quotas['domains'].get(postDomain):
if quotas['domains'][postDomain]>domainMaxPostsPerDay:
queue.pop(0)
continue
quotas['domains'][postDomain]+=1
else:
quotas['domains'][postDomain]=1
if accountMaxPostsPerDay>0:
postHandle=queueJson['postNickname']+'@'+postDomain
if quotas['accounts'].get(postHandle):
if quotas['accounts'][postHandle]>accountMaxPostsPerDay:
queue.pop(0)
continue
quotas['accounts'][postHandle]+=1
else:
quotas['accounts'][postHandle]=1
2019-07-15 10:22:19 +00:00
if debug:
2019-07-15 10:25:13 +00:00
if accountMaxPostsPerDay>0 or domainMaxPostsPerDay>0:
pprint(quotas)
2019-07-15 10:22:19 +00:00
2019-07-04 19:34:28 +00:00
# Try a few times to obtain the public key
2019-07-04 12:23:53 +00:00
pubKey=None
keyId=None
2019-07-04 17:31:41 +00:00
for tries in range(8):
2019-07-28 13:30:19 +00:00
if queueJson['postFromWebInterface']:
break
2019-07-04 14:36:29 +00:00
keyId=None
signatureParams=queueJson['headers'].split(',')
for signatureItem in signatureParams:
if signatureItem.startswith('keyId='):
if '"' in signatureItem:
keyId=signatureItem.split('"')[1]
break
if not keyId:
if debug:
print('DEBUG: No keyId in signature: '+queueJson['headers']['signature'])
os.remove(queueFilename)
queue.pop(0)
continue
pubKey=getPersonPubKey(session,keyId,personCache,debug)
2019-07-04 17:31:41 +00:00
if pubKey:
print('DEBUG: public key: '+str(pubKey))
break
if debug:
print('DEBUG: Retry '+str(tries+1)+' obtaining public key for '+keyId)
time.sleep(5)
2019-07-28 13:30:19 +00:00
if not queueJson['postFromWebInterface']:
if not pubKey:
if debug:
print('DEBUG: public key could not be obtained from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-04 12:23:53 +00:00
2019-07-28 13:30:19 +00:00
# check the signature
verifyHeaders={
'host': queueJson['host'],
'signature': queueJson['headers']
}
if not verifyPostHeaders(httpPrefix, \
pubKey, verifyHeaders, \
queueJson['path'], False, \
json.dumps(queueJson['post'])):
if debug:
print('DEBUG: Header signature check failed')
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-04 12:23:53 +00:00
2019-07-28 13:30:19 +00:00
if debug:
print('DEBUG: Signature check success')
2019-07-04 17:31:41 +00:00
2019-07-17 10:34:00 +00:00
if receiveUndo(session, \
baseDir,httpPrefix,port, \
sendThreads,postLog, \
cachedWebfingers,
personCache,
queueJson['post'], \
federationList, \
debug, \
acceptedCaps=["inbox:write","objects:read"]):
if debug:
print('DEBUG: Undo accepted from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-05 18:57:19 +00:00
if receiveFollowRequest(session, \
baseDir,httpPrefix,port, \
sendThreads,postLog, \
cachedWebfingers,
personCache,
2019-07-04 20:25:19 +00:00
queueJson['post'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-09 17:54:08 +00:00
debug, \
acceptedCaps=["inbox:write","objects:read"]):
2019-07-04 12:23:53 +00:00
if debug:
2019-07-04 17:31:41 +00:00
print('DEBUG: Follow accepted from '+keyId)
2019-07-05 18:57:19 +00:00
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-06 15:17:21 +00:00
if receiveAcceptReject(session, \
2019-07-06 19:24:52 +00:00
baseDir,httpPrefix,domain,port, \
2019-07-06 15:17:21 +00:00
sendThreads,postLog, \
cachedWebfingers,
personCache,
queueJson['post'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-06 15:17:21 +00:00
debug):
if debug:
print('DEBUG: Accept/Reject received from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-09 14:20:23 +00:00
if receiveUpdate(session, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers,
personCache,
queueJson['post'], \
federationList, \
debug):
if debug:
print('DEBUG: Update accepted from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
# get recipients list
recipientsDict,recipientsDictFollowers= \
2019-07-11 12:29:31 +00:00
inboxPostRecipients(baseDir,queueJson['post'],httpPrefix,domain,port,debug)
if len(recipientsDict.items())==0 and \
len(recipientsDictFollowers.items())==0:
if debug:
pprint(queueJson['post'])
print('DEBUG: no recipients were resolved for post arriving in inbox')
os.remove(queueFilename)
queue.pop(0)
continue
# if there are only a small number of followers then process them as if they
# were specifically addresses to particular accounts
noOfFollowItems=len(recipientsDictFollowers.items())
if noOfFollowItems>0:
if noOfFollowItems<5:
if debug:
print('DEBUG: moving '+str(noOfFollowItems)+' inbox posts addressed to followers')
for handle,postItem in recipientsDictFollowers.items():
2019-07-11 12:29:31 +00:00
recipientsDict[handle]=postItem
recipientsDictFollowers={}
recipientsList=[recipientsDict,recipientsDictFollowers]
if debug:
print('*************************************')
print('Resolved recipients list:')
pprint(recipientsDict)
2019-07-11 12:29:31 +00:00
print('Resolved followers list:')
pprint(recipientsDictFollowers)
print('*************************************')
2019-07-08 23:05:48 +00:00
if queueJson['post'].get('capability'):
if not isinstance(queueJson['post']['capability'], list):
if debug:
2019-07-08 23:05:48 +00:00
print('DEBUG: capability on post should be a list')
os.remove(queueFilename)
queue.pop(0)
continue
# Copy any posts addressed to followers into the shared inbox
# this avoid copying file multiple times to potentially many
# individual inboxes
# This obviously bypasses object capabilities and so
# any checking will needs to be handled at the time when inbox
# GET happens on individual accounts.
# See posts.py/createBoxBase
2019-07-18 09:31:29 +00:00
if len(recipientsDictFollowers)>0:
with open(queueJson['destination'].replace(inboxHandle,inboxHandle), 'w') as fp:
commentjson.dump(queueJson['post'], fp, indent=4, sort_keys=False)
# for posts addressed to specific accounts
for handle,capsId in recipientsDict.items():
destination=queueJson['destination'].replace(inboxHandle,handle)
# check that capabilities are accepted
2019-07-08 23:05:48 +00:00
if queueJson['post'].get('capability'):
capabilityIdList=queueJson['post']['capability']
# does the capability id list within the post contain the id
# of the recipient with this handle?
# Here the capability id begins with the handle, so this could also
# be matched separately, but it's probably not necessary
2019-07-08 23:05:48 +00:00
if capsId in capabilityIdList:
inboxAfterCapabilities(session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \
cachedWebfingers, \
personCache,queue,domain, \
port,useTor, \
federationList,ocapAlways, \
debug,acceptedCaps, \
2019-07-13 21:00:12 +00:00
queueFilename,destination, \
maxReplies,allowDeletion)
2019-07-08 23:05:48 +00:00
else:
if debug:
print('DEBUG: object capabilities check failed')
pprint(queueJson['post'])
else:
if not ocapAlways:
inboxAfterCapabilities(session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \
cachedWebfingers, \
personCache,queue,domain, \
port,useTor, \
federationList,ocapAlways, \
debug,acceptedCaps, \
2019-07-13 21:00:12 +00:00
queueFilename,destination, \
maxReplies,allowDeletion)
2019-07-09 08:44:24 +00:00
if debug:
print('DEBUG: object capabilities check failed')
2019-07-08 23:05:48 +00:00
if debug:
print('DEBUG: Queue post accepted')
2019-07-08 23:05:48 +00:00
os.remove(queueFilename)
2019-07-04 12:23:53 +00:00
queue.pop(0)