epicyon/inbox.py

__filename__ = "inbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "0.0.1"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"

import json
import os
import datetime
import time
import json
import commentjson
from shutil import copyfile
from utils import urlPermitted
from utils import createInboxQueueDir
from utils import getStatusNumber
from httpsig import verifyPostHeaders
from session import createSession
from session import getJson
from follow import receiveFollowRequest
from pprint import pprint
from cache import getPersonFromCache
from cache import storePersonInCache
from acceptreject import receiveAcceptReject
from capabilities import getOcapFilename
from capabilities import CapablePost

def getPersonPubKey(session,personUrl: str,personCache: {},debug: bool) -> str:
    if not personUrl:
        return None
    personUrl=personUrl.replace('#main-key','')
    personJson = getPersonFromCache(personUrl,personCache)
    if not personJson:
        if debug:
            print('DEBUG: Obtaining public key for '+personUrl)
        asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
        personJson = getJson(session,personUrl,asHeader,None)
        if not personJson:
            return None
    pubKey=None
    if personJson.get('publicKey'):
        if personJson['publicKey'].get('publicKeyPem'):
            pubKey=personJson['publicKey']['publicKeyPem']
    else:
        if personJson.get('publicKeyPem'):
            pubKey=personJson['publicKeyPem']

    if not pubKey:
        if debug:
            print('DEBUG: Public key not found for '+personUrl)

    storePersonInCache(personUrl,personJson,personCache)
    return pubKey

def inboxMessageHasParams(messageJson: {}) -> bool:
    """Checks whether an incoming message contains expected parameters
    """
    expectedParams=['type','actor','object']
    for param in expectedParams:
        if not messageJson.get(param):
            return False
    if not messageJson.get('to'):
        allowedWithoutToParam=['Follow','Request','Capability']
        if messageJson['type'] not in allowedWithoutToParam:
            return False
    return True

def inboxPermittedMessage(domain: str,messageJson: {},federationList: [],ocapGranted: {}) -> bool:
    """ check that we are receiving from a permitted domain
    """
    testParam='actor'
    if not messageJson.get(testParam):
        return False
    actor=messageJson[testParam]
    # always allow the local domain
    if domain in actor:
        return True

    if not urlPermitted(actor,federationList,ocapGranted,"inbox:write"):
        return False

    if messageJson['type']!='Follow':
        if messageJson.get('object'):
            if messageJson['object'].get('inReplyTo'):
                inReplyTo=messageJson['object']['inReplyTo']
                if not urlPermitted(inReplyTo,federationList,ocapGranted):
                    return False

    return True

def validPublishedDate(published) -> bool:
    currTime=datetime.datetime.utcnow()
    pubDate=datetime.datetime.strptime(published,"%Y-%m-%dT%H:%M:%SZ")
    daysSincePublished = (currTime - pubTime).days
    if daysSincePublished>30:
        return False
    return True

def savePostToInboxQueue(baseDir: str,httpPrefix: str,nickname: str, domain: str,postJson: {},host: str,headers: str,postPath: str,debug: bool) -> str:
    """Saves the give json to the inbox queue for the person
    keyId specifies the actor sending the post
    """
    if ':' in domain:
        domain=domain.split(':')[0]
    if postJson.get('id'):
        postId=postJson['id'].replace('/activity','')
    else:
        statusNumber,published = getStatusNumber()
        postId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
    
    currTime=datetime.datetime.utcnow()
    published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")

    inboxQueueDir=createInboxQueueDir(nickname,domain,baseDir)

    handle=nickname+'@'+domain
    destination=baseDir+'/accounts/'+handle+'/inbox/'+postId.replace('/','#')+'.json'
    if os.path.isfile(destination):
        if debug:
            print('DEBUG: inbox item already exists')
        return None
    filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json'

    sharedInboxItem=False
    if nickname=='inbox':
        sharedInboxItem=True
        
    newQueueItem = {
        'nickname': nickname,
        'domain': domain,
        'sharedInbox': sharedInboxItem,
        'published': published,
        'host': host,
        'headers': headers,
        'path': postPath,
        'post': postJson,
        'filename': filename,
        'destination': destination
    }

    if debug:
        print('Inbox queue item created')
        pprint(newQueueItem)
    
    with open(filename, 'w') as fp:
        commentjson.dump(newQueueItem, fp, indent=4, sort_keys=False)
    return filename

def runInboxQueue(baseDir: str,httpPrefix: str,sendThreads: [],postLog: [],cachedWebfingers: {},personCache: {},queue: [],domain: str,port: int,useTor: bool,federationList: [],ocapAlways: bool,ocapGranted: {},debug: bool) -> None:
    """Processes received items and moves them to
    the appropriate directories
    """
    currSessionTime=int(time.time())
    sessionLastUpdate=currSessionTime
    session=createSession(domain,port,useTor)
    if debug:
        print('DEBUG: Inbox queue running')

    while True:
        time.sleep(1)
        if len(queue)>0:
            currSessionTime=int(time.time())
            if currSessionTime-sessionLastUpdate>1200:
                session=createSession(domain,port,useTor)
                sessionLastUpdate=currSessionTime

            # oldest item first
            queue.sort()
            queueFilename=queue[0]
            if not os.path.isfile(queueFilename):
                if debug:
                    print("DEBUG: queue item rejected becase it has no file: "+queueFilename)
                queue.pop(0)
                continue

            # Load the queue json
            with open(queueFilename, 'r') as fp:
                queueJson=commentjson.load(fp)

            sentToSharedInbox=False
            if queueJson['post'].get('actor'):
                if queueJson['post']['actor'].endswith('/inbox'):
                    sentToSharedInbox=True

            if sentToSharedInbox:
                # if this is arriving at the shared inbox then
                # don't do the capabilities checks
                capabilitiesPassed=True
                # TODO how to handle capabilities in the shared inbox scenario?
                # should 'capability' be a list instead of a single value?
            else:
                # check that capabilities are accepted            
                capabilitiesPassed=False
                if queueJson['post'].get('capability'):
                    if not isinstance(queueJson['post']['capability'], list):
                        if debug:
                            print('DEBUG: capability on post should be a list')
                        os.remove(queueFilename)
                        queue.pop(0)
                        continue
                    capabilityIdList=queueJson['post']['capability']
                    
                    ocapFilename= \
                        getOcapFilename(baseDir, \
                                        queueJson['nickname'],queueJson['domain'], \
                                        queueJson['post']['actor'],'accept')
                    if not os.path.isfile(ocapFilename):
                        if debug:
                            print('DEBUG: capabilities for '+ \
                                  queueJson['post']['actor']+' do not exist')
                        os.remove(queueFilename)
                        queue.pop(0)
                        continue                
                    with open(ocapFilename, 'r') as fp:
                        oc=commentjson.load(fp)
                        if not oc.get('id'):
                            if debug:
                                print('DEBUG: capabilities for '+queueJson['post']['actor']+' do not contain an id')
                            os.remove(queueFilename)
                            queue.pop(0)
                            continue
                        if oc['id']!=capabilityIdList[0]:
                            if debug:
                                print('DEBUG: capability id mismatch')
                            os.remove(queueFilename)
                            queue.pop(0)
                            continue
                        if not oc.get('capability'):
                            if debug:
                                print('DEBUG: missing capability list')
                            os.remove(queueFilename)
                            queue.pop(0)
                            continue
                        if not CapablePost(queueJson['post'],oc['capability'],debug):
                            if debug:
                                print('DEBUG: insufficient capabilities to write to inbox from '+ \
                                      queueJson['post']['actor'])
                            os.remove(queueFilename)
                            queue.pop(0)
                            continue
                        if debug:
                            print('DEBUG: object capabilities check success')
                        capabilitiesPassed=True

            if ocapAlways and not capabilitiesPassed:
                # Allow follow types through
                # i.e. anyone can make a follow request
                if queueJson['post'].get('type'):
                    if queueJson['post']['type']=='Follow' or \
                       queueJson['post']['type']=='Accept':
                        capabilitiesPassed=True
                if not capabilitiesPassed:
                    if debug:
                        print('DEBUG: object capabilities check failed')
                        pprint(queueJson['post'])
                    os.remove(queueFilename)
                    queue.pop(0)
                    continue

            # Try a few times to obtain the public key
            pubKey=None
            keyId=None
            for tries in range(8):
                keyId=None
                signatureParams=queueJson['headers'].split(',')
                for signatureItem in signatureParams:
                    if signatureItem.startswith('keyId='):
                        if '"' in signatureItem:
                            keyId=signatureItem.split('"')[1]
                            break
                if not keyId:
                    if debug:
                        print('DEBUG: No keyId in signature: '+queueJson['headers']['signature'])
                    os.remove(queueFilename)
                    queue.pop(0)
                    continue

                pubKey=getPersonPubKey(session,keyId,personCache,debug)
                if pubKey:
                    print('DEBUG: public key: '+str(pubKey))
                    break
                    
                if debug:
                    print('DEBUG: Retry '+str(tries+1)+' obtaining public key for '+keyId)
                time.sleep(5)

            if not pubKey:
                if debug:
                    print('DEBUG: public key could not be obtained from '+keyId)
                os.remove(queueFilename)
                queue.pop(0)
                continue

            # check the signature
            verifyHeaders={
                'host': queueJson['host'],
                'signature': queueJson['headers']
            }            
            if not verifyPostHeaders(httpPrefix, \
                                     pubKey, verifyHeaders, \
                                     queueJson['path'], False, \
                                     json.dumps(queueJson['post'])):
                if debug:
                    print('DEBUG: Header signature check failed')
                os.remove(queueFilename)
                queue.pop(0)
                continue

            if debug:
                print('DEBUG: Signature check success')

            if receiveFollowRequest(session, \
                                    baseDir,httpPrefix,port, \
                                    sendThreads,postLog, \
                                    cachedWebfingers,
                                    personCache,
                                    queueJson['post'], \
                                    federationList,ocapGranted, \
                                    debug):
                if debug:
                    print('DEBUG: Follow accepted from '+keyId)
                os.remove(queueFilename)
                queue.pop(0)
                continue

            if receiveAcceptReject(session, \
                                   baseDir,httpPrefix,domain,port, \
                                   sendThreads,postLog, \
                                   cachedWebfingers,
                                   personCache,
                                   queueJson['post'], \
                                   federationList,ocapGranted, \
                                   debug):
                if debug:
                    print('DEBUG: Accept/Reject received from '+keyId)
                os.remove(queueFilename)
                queue.pop(0)
                continue

            if debug:
                print('DEBUG: Queue post accepted')

            if queueJson['sharedInbox']:
                if '/users/' in keyId:
                    # Who is this from? Use the actor from the keyId where we obtained the public key
                    fromList=keyId.replace('https://','').replace('http://','').replace('dat://','').replace('#main-key','').split('/users/')
                    fromNickname=fromList[1]
                    fromDomain=fromList[0]
                    # get the followers of the sender
                    followList=getFollowersOfPerson(baseDir,fromNickname,fromDomain)
                    for followerHandle in followList:
                        followerDir=baseDir+'/accounts/'+followerHandle
                        if os.path.isdir(followerDir):                        
                            if not os.path.isdir(followerDir+'/inbox'):
                                os.mkdir(followerDir+'/inbox')
                                postId=queueJson['post']['id'].replace('/activity','')
                                destination=followerDir+'/inbox/'+postId.replace('/','#')+'.json'
                                if os.path.isfile(destination):
                                    # post already exists in this person's inbox
                                    continue
                                # We could do this in a more storage space efficient way
                                # by linking to the inbox of inbox@domain
                                # However, this allows for easy deletion by individuals
                                # without affecting any other people
                                copyfile(queueFilename, destination)
                # copy to followers
                # remove item from shared inbox
                os.remove(queueFilename)
            else:
                # move to the destination inbox
                os.rename(queueFilename,queueJson['destination'])
            queue.pop(0)