epicyon/inbox.py

1964 lines
79 KiB
Python
Raw Normal View History

2019-06-28 21:59:54 +00:00
__filename__ = "inbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
2019-08-29 13:35:29 +00:00
__version__ = "1.0.0"
2019-06-28 21:59:54 +00:00
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
import json
import os
2019-06-29 10:08:59 +00:00
import datetime
2019-07-04 12:23:53 +00:00
import time
import json
import commentjson
from shutil import copyfile
2019-07-02 10:39:55 +00:00
from utils import urlPermitted
2019-07-04 10:02:56 +00:00
from utils import createInboxQueueDir
2019-07-06 13:49:25 +00:00
from utils import getStatusNumber
2019-07-09 14:20:23 +00:00
from utils import getDomainFromActor
from utils import getNicknameFromActor
from utils import domainPermitted
2019-07-11 12:29:31 +00:00
from utils import locatePost
2019-07-14 16:37:01 +00:00
from utils import deletePost
2019-07-14 16:57:06 +00:00
from utils import removeAttachment
2019-08-12 18:02:29 +00:00
from utils import removeModerationPostFromIndex
2019-07-04 12:23:53 +00:00
from httpsig import verifyPostHeaders
from session import createSession
2019-07-04 19:34:28 +00:00
from session import getJson
2019-07-04 12:23:53 +00:00
from follow import receiveFollowRequest
2019-07-08 18:55:39 +00:00
from follow import getFollowersOfActor
2019-07-17 11:54:13 +00:00
from follow import unfollowerOfPerson
2019-07-04 14:36:29 +00:00
from pprint import pprint
2019-07-04 19:34:28 +00:00
from cache import getPersonFromCache
2019-07-04 20:25:19 +00:00
from cache import storePersonInCache
2019-07-06 15:17:21 +00:00
from acceptreject import receiveAcceptReject
2019-07-07 15:51:04 +00:00
from capabilities import getOcapFilename
2019-07-07 22:06:46 +00:00
from capabilities import CapablePost
2019-07-09 14:20:23 +00:00
from capabilities import capabilitiesReceiveUpdate
2019-07-10 12:40:31 +00:00
from like import updateLikesCollection
2019-07-12 09:10:09 +00:00
from like import undoLikesCollectionEntry
from blocking import isBlocked
2019-07-14 20:50:27 +00:00
from filters import isFiltered
from announce import updateAnnounceCollection
from httpsig import messageContentDigest
from blocking import isBlockedDomain
from posts import downloadAnnounce
from posts import isDM
from posts import isReply
2019-10-04 12:39:46 +00:00
from posts import sendSignedJson
2019-07-04 19:34:28 +00:00
def validInbox(baseDir: str,nickname: str,domain: str) -> bool:
2019-07-18 11:35:48 +00:00
"""Checks whether files were correctly saved to the inbox
"""
if ':' in domain:
domain=domain.split(':')[0]
inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
if not os.path.isdir(inboxDir):
return True
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
filename = os.path.join(subdir, f)
if not os.path.isfile(filename):
print('filename: '+filename)
return False
if 'postNickname' in open(filename).read():
2019-07-18 11:35:48 +00:00
print('queue file incorrectly saved to '+filename)
return False
return True
def validInboxFilenames(baseDir: str,nickname: str,domain: str, \
expectedDomain: str,expectedPort: int) -> bool:
"""Used by unit tests to check that the port number gets appended to
domain names within saved post filenames
"""
if ':' in domain:
domain=domain.split(':')[0]
inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
if not os.path.isdir(inboxDir):
return True
expectedStr=expectedDomain+':'+str(expectedPort)
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
filename = os.path.join(subdir, f)
if not os.path.isfile(filename):
print('filename: '+filename)
return False
if not expectedStr in filename:
2019-08-16 13:47:01 +00:00
print('Expected: '+expectedStr)
2019-07-18 11:35:48 +00:00
print('Invalid filename: '+filename)
return False
return True
2019-08-20 09:16:03 +00:00
def getPersonPubKey(baseDir: str,session,personUrl: str, \
personCache: {},debug: bool, \
2019-08-14 20:12:27 +00:00
projectVersion: str,httpPrefix: str,domain: str) -> str:
2019-07-04 19:34:28 +00:00
if not personUrl:
return None
personUrl=personUrl.replace('#main-key','')
2019-08-05 16:05:08 +00:00
if personUrl.endswith('/users/inbox'):
if debug:
print('DEBUG: Obtaining public key for shared inbox')
personUrl=personUrl.replace('/users/inbox','/inbox')
2019-08-20 09:37:09 +00:00
personJson = getPersonFromCache(baseDir,personUrl,personCache)
2019-07-04 19:34:28 +00:00
if not personJson:
if debug:
print('DEBUG: Obtaining public key for '+personUrl)
2019-09-01 12:09:29 +00:00
asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
2019-08-14 20:12:27 +00:00
personJson = getJson(session,personUrl,asHeader,None,projectVersion,httpPrefix,domain)
2019-07-04 19:34:28 +00:00
if not personJson:
return None
pubKey=None
if personJson.get('publicKey'):
if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem']
else:
if personJson.get('publicKeyPem'):
pubKey=personJson['publicKeyPem']
if not pubKey:
if debug:
print('DEBUG: Public key not found for '+personUrl)
2019-08-20 09:16:03 +00:00
storePersonInCache(baseDir,personUrl,personJson,personCache)
2019-07-04 19:34:28 +00:00
return pubKey
2019-06-28 21:59:54 +00:00
2019-07-02 15:07:27 +00:00
def inboxMessageHasParams(messageJson: {}) -> bool:
"""Checks whether an incoming message contains expected parameters
"""
2019-07-06 13:49:25 +00:00
expectedParams=['type','actor','object']
2019-07-02 15:07:27 +00:00
for param in expectedParams:
if not messageJson.get(param):
return False
2019-07-06 13:49:25 +00:00
if not messageJson.get('to'):
2019-08-18 16:49:35 +00:00
allowedWithoutToParam=['Like','Follow','Request','Accept','Capability','Undo']
2019-07-06 13:49:25 +00:00
if messageJson['type'] not in allowedWithoutToParam:
return False
2019-07-02 15:07:27 +00:00
return True
2019-07-09 14:20:23 +00:00
def inboxPermittedMessage(domain: str,messageJson: {},federationList: []) -> bool:
2019-06-28 21:59:54 +00:00
""" check that we are receiving from a permitted domain
"""
testParam='actor'
if not messageJson.get(testParam):
return False
actor=messageJson[testParam]
# always allow the local domain
2019-07-01 11:48:54 +00:00
if domain in actor:
2019-06-28 21:59:54 +00:00
return True
2019-07-09 14:20:23 +00:00
if not urlPermitted(actor,federationList,"inbox:write"):
2019-06-28 21:59:54 +00:00
return False
2019-07-11 17:55:10 +00:00
if messageJson['type']!='Follow' and \
messageJson['type']!='Like' and \
2019-07-11 21:38:28 +00:00
messageJson['type']!='Delete' and \
2019-07-11 17:55:10 +00:00
messageJson['type']!='Announce':
2019-07-06 13:49:25 +00:00
if messageJson.get('object'):
2019-07-15 09:20:16 +00:00
if not isinstance(messageJson['object'], dict):
return False
2019-07-06 13:49:25 +00:00
if messageJson['object'].get('inReplyTo'):
inReplyTo=messageJson['object']['inReplyTo']
2019-08-05 19:30:27 +00:00
if not urlPermitted(inReplyTo,federationList,"inbox:write"):
2019-07-06 13:49:25 +00:00
return False
2019-06-28 21:59:54 +00:00
return True
2019-06-29 10:08:59 +00:00
2019-10-11 16:54:55 +00:00
def validPublishedDate(published: str) -> bool:
2019-06-29 10:08:59 +00:00
currTime=datetime.datetime.utcnow()
pubDate=datetime.datetime.strptime(published,"%Y-%m-%dT%H:%M:%SZ")
daysSincePublished = (currTime - pubTime).days
if daysSincePublished>30:
return False
return True
2019-07-04 10:02:56 +00:00
2019-08-18 09:39:12 +00:00
def savePostToInboxQueue(baseDir: str,httpPrefix: str, \
nickname: str, domain: str, \
postJsonObject: {}, \
messageBytes: str, \
httpHeaders: {}, \
postPath: str,debug: bool) -> str:
2019-07-04 10:02:56 +00:00
"""Saves the give json to the inbox queue for the person
keyId specifies the actor sending the post
"""
2019-07-18 11:35:48 +00:00
originalDomain=domain
2019-07-04 10:02:56 +00:00
if ':' in domain:
domain=domain.split(':')[0]
# block at the ealiest stage possible, which means the data
# isn't written to file
2019-07-15 10:22:19 +00:00
postNickname=None
postDomain=None
2019-08-16 09:35:06 +00:00
actor=None
if postJsonObject.get('actor'):
2019-08-16 09:35:06 +00:00
actor=postJsonObject['actor']
postNickname=getNicknameFromActor(postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
if not postNickname:
2019-09-02 09:43:43 +00:00
print('No post Nickname in actor '+postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
return None
2019-09-02 09:43:43 +00:00
postDomain,postPort=getDomainFromActor(postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
if not postDomain:
pprint(postJsonObject)
print('No post Domain in actor')
return None
2019-08-18 09:39:12 +00:00
if isBlocked(baseDir,nickname,domain,postNickname,postDomain):
if debug:
print('DEBUG: post from '+postNickname+' blocked')
return None
2019-07-15 10:22:19 +00:00
if postPort:
if postPort!=80 and postPort!=443:
if ':' not in postDomain:
postDomain=postDomain+':'+str(postPort)
2019-07-14 20:50:27 +00:00
2019-08-05 09:28:12 +00:00
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('inReplyTo'):
if isinstance(postJsonObject['object']['inReplyTo'], str):
replyNickname=getNicknameFromActor(postJsonObject['object']['inReplyTo'])
replyDomain,replyPort=getDomainFromActor(postJsonObject['object']['inReplyTo'])
if replyNickname and replyDomain:
2019-09-03 11:27:21 +00:00
if isBlocked(baseDir,nickname,domain,replyNickname,replyDomain):
2019-09-01 20:00:45 +00:00
print('WARN: post contains reply to a blocked account: '+replyNickname+'@'+replyDomain)
return None
else:
print('WARN: post is a reply to an unidentified account: '+postJsonObject['object']['inReplyTo'])
return None
2019-08-05 09:28:12 +00:00
if postJsonObject['object'].get('content'):
if isinstance(postJsonObject['object']['content'], str):
if isFiltered(baseDir,nickname,domain,postJsonObject['object']['content']):
2019-09-01 19:54:02 +00:00
print('WARN: post was filtered out due to content')
2019-08-05 09:28:12 +00:00
return None
2019-08-16 09:35:06 +00:00
originalPostId=None
2019-07-14 16:57:06 +00:00
if postJsonObject.get('id'):
2019-08-16 19:48:32 +00:00
originalPostId=postJsonObject['id'].replace('/activity','').replace('/undo','')
2019-08-16 15:04:40 +00:00
currTime=datetime.datetime.utcnow()
postId=None
if postJsonObject.get('id'):
#if '/statuses/' not in postJsonObject['id']:
2019-08-16 19:48:32 +00:00
postId=postJsonObject['id'].replace('/activity','').replace('/undo','')
2019-08-16 15:04:40 +00:00
published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
if not postId:
statusNumber,published = getStatusNumber()
if actor:
postId=actor+'/statuses/'+statusNumber
else:
postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
2019-08-16 09:35:06 +00:00
2019-08-16 13:47:01 +00:00
# NOTE: don't change postJsonObject['id'] before signature check
2019-07-06 13:49:25 +00:00
2019-07-05 11:27:18 +00:00
inboxQueueDir=createInboxQueueDir(nickname,domain,baseDir)
handle=nickname+'@'+domain
destination=baseDir+'/accounts/'+handle+'/inbox/'+postId.replace('/','#')+'.json'
2019-08-18 12:06:08 +00:00
#if os.path.isfile(destination):
# if debug:
# print(destination)
# print('DEBUG: inbox item already exists')
# return None
filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json'
sharedInboxItem=False
2019-07-08 13:30:04 +00:00
if nickname=='inbox':
nickname=originalDomain
sharedInboxItem=True
2019-07-04 14:36:29 +00:00
newQueueItem = {
2019-08-16 09:35:06 +00:00
'originalId': originalPostId,
2019-07-15 09:20:16 +00:00
'id': postId,
2019-08-16 09:35:06 +00:00
'actor': actor,
2019-07-07 15:51:04 +00:00
'nickname': nickname,
'domain': domain,
2019-07-15 10:22:19 +00:00
'postNickname': postNickname,
'postDomain': postDomain,
'sharedInbox': sharedInboxItem,
2019-07-04 10:09:27 +00:00
'published': published,
2019-08-15 21:34:25 +00:00
'httpHeaders': httpHeaders,
2019-07-05 22:13:20 +00:00
'path': postPath,
2019-07-14 16:57:06 +00:00
'post': postJsonObject,
'digest': messageContentDigest(messageBytes),
'filename': filename,
2019-08-05 09:50:45 +00:00
'destination': destination
2019-07-04 10:02:56 +00:00
}
2019-07-06 13:49:25 +00:00
if debug:
print('Inbox queue item created')
pprint(newQueueItem)
2019-09-30 22:39:02 +00:00
try:
with open(filename, 'w') as fp:
commentjson.dump(newQueueItem, fp, indent=4, sort_keys=False)
except Exception as e:
print(e)
2019-07-04 10:02:56 +00:00
return filename
2019-07-04 12:23:53 +00:00
2019-07-08 18:55:39 +00:00
def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \
actor: str,queue: [],queueJson: {}, \
capabilityId: str,debug : bool) -> bool:
if nickname=='inbox':
return True
ocapFilename= \
getOcapFilename(baseDir, \
queueJson['nickname'],queueJson['domain'], \
actor,'accept')
2019-08-18 20:47:12 +00:00
if not ocapFilename:
return False
2019-07-08 18:55:39 +00:00
if not os.path.isfile(ocapFilename):
if debug:
print('DEBUG: capabilities for '+ \
actor+' do not exist')
os.remove(queueFilename)
queue.pop(0)
return False
2019-10-11 18:03:58 +00:00
tries=0
oc=None
while tries<5:
try:
with open(ocapFilename, 'r') as fp:
oc=commentjson.load(fp)
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
if not oc:
return False
2019-07-08 18:55:39 +00:00
if not oc.get('id'):
if debug:
print('DEBUG: capabilities for '+actor+' do not contain an id')
os.remove(queueFilename)
queue.pop(0)
return False
if oc['id']!=capabilityId:
if debug:
print('DEBUG: capability id mismatch')
os.remove(queueFilename)
queue.pop(0)
return False
if not oc.get('capability'):
if debug:
print('DEBUG: missing capability list')
os.remove(queueFilename)
queue.pop(0)
return False
if not CapablePost(queueJson['post'],oc['capability'],debug):
if debug:
print('DEBUG: insufficient capabilities to write to inbox from '+actor)
os.remove(queueFilename)
queue.pop(0)
return False
if debug:
print('DEBUG: object capabilities check success')
return True
2019-07-08 22:12:24 +00:00
def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \
recipientsDict :{}, \
domainMatch: str,domain :str, \
2019-07-11 12:29:31 +00:00
actor :str,debug: bool) -> bool:
2019-07-08 22:12:24 +00:00
"""Given a list of post recipients (toList) from 'to' or 'cc' parameters
populate a recipientsDict with the handle and capabilities id for each
"""
followerRecipients=False
for recipient in toList:
2019-09-03 19:53:22 +00:00
if not recipient:
continue
2019-07-08 22:12:24 +00:00
# is this a to a local account?
if domainMatch in recipient:
# get the handle for the local account
nickname=recipient.split(domainMatch)[1]
handle=nickname+'@'+domain
if os.path.isdir(baseDir+'/accounts/'+handle):
# are capabilities granted for this account to the
# sender (actor) of the post?
ocapFilename=baseDir+'/accounts/'+handle+'/ocap/accept/'+actor.replace('/','#')+'.json'
if os.path.isfile(ocapFilename):
# read the granted capabilities and obtain the id
loadedOcap=False
2019-10-11 18:03:58 +00:00
tries=0
while tries<5:
try:
with open(ocapFilename, 'r') as fp:
ocapJson=commentjson.load(fp)
loadedOcap=True
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
if loadedOcap:
2019-07-08 22:12:24 +00:00
if ocapJson.get('id'):
# append with the capabilities id
recipientsDict[handle]=ocapJson['id']
else:
recipientsDict[handle]=None
else:
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: '+ocapFilename+' not found')
2019-07-08 22:12:24 +00:00
recipientsDict[handle]=None
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: '+baseDir+'/accounts/'+handle+' does not exist')
else:
if debug:
print('DEBUG: '+recipient+' is not local to '+domainMatch)
print(str(toList))
2019-07-08 22:12:24 +00:00
if recipient.endswith('followers'):
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: followers detected as post recipients')
2019-07-08 22:12:24 +00:00
followerRecipients=True
return followerRecipients,recipientsDict
2019-07-11 12:29:31 +00:00
def inboxPostRecipients(baseDir :str,postJsonObject :{},httpPrefix :str,domain : str,port :int, debug :bool) -> ([],[]):
"""Returns dictionaries containing the recipients of the given post
The shared dictionary contains followers
"""
2019-07-08 22:12:24 +00:00
recipientsDict={}
recipientsDictFollowers={}
2019-07-08 22:12:24 +00:00
if not postJsonObject.get('actor'):
2019-07-11 12:29:31 +00:00
if debug:
pprint(postJsonObject)
print('WARNING: inbox post has no actor')
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
if ':' in domain:
domain=domain.split(':')[0]
domainBase=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
2019-07-08 22:12:24 +00:00
domainMatch='/'+domain+'/users/'
actor = postJsonObject['actor']
# first get any specific people which the post is addressed to
followerRecipients=False
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('to'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['object']['to'], list):
recipientsList=postJsonObject['object']['to']
else:
recipientsList=[postJsonObject['object']['to']]
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: resolving "to"')
2019-07-08 22:12:24 +00:00
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
2019-08-16 17:51:00 +00:00
recipientsList, \
2019-07-08 22:12:24 +00:00
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no "to"')
2019-07-08 22:12:24 +00:00
if postJsonObject['object'].get('cc'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['object']['cc'], list):
recipientsList=postJsonObject['object']['cc']
else:
recipientsList=[postJsonObject['object']['cc']]
2019-07-08 22:12:24 +00:00
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
2019-08-16 17:51:00 +00:00
recipientsList, \
2019-07-08 22:12:24 +00:00
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no cc')
else:
if debug:
if isinstance(postJsonObject['object'], str):
if '/statuses/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to a post')
else:
if '/users/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to an actor')
2019-07-08 22:12:24 +00:00
if postJsonObject.get('to'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['to'], list):
recipientsList=postJsonObject['to']
else:
recipientsList=[postJsonObject['to']]
2019-07-08 22:12:24 +00:00
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
2019-08-16 17:51:00 +00:00
recipientsList, \
2019-07-08 22:12:24 +00:00
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
if postJsonObject.get('cc'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['cc'], list):
recipientsList=postJsonObject['cc']
else:
recipientsList=[postJsonObject['cc']]
2019-07-08 22:12:24 +00:00
includesFollowers,recipientsDict= \
inboxPostRecipientsAdd(baseDir,httpPrefix, \
2019-08-16 17:51:00 +00:00
recipientsList, \
2019-07-08 22:12:24 +00:00
recipientsDict, \
2019-07-11 12:29:31 +00:00
domainMatch,domainBase, \
actor,debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
followerRecipients=True
if not followerRecipients:
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: no followers were resolved')
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
# now resolve the followers
recipientsDictFollowers= \
2019-07-11 12:29:31 +00:00
getFollowersOfActor(baseDir,actor,debug)
2019-07-08 22:12:24 +00:00
return recipientsDict,recipientsDictFollowers
2019-07-08 22:12:24 +00:00
2019-07-17 10:34:00 +00:00
def receiveUndoFollow(session,baseDir: str,httpPrefix: str, \
2019-07-17 10:38:10 +00:00
port: int,messageJson: {}, \
federationList: [], \
debug : bool) -> bool:
2019-07-17 10:34:00 +00:00
if not messageJson['object'].get('actor'):
if debug:
print('DEBUG: follow request has no actor within object')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['object']['actor'] and '/profile/' not in messageJson['object']['actor']:
2019-07-17 10:34:00 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor within object')
2019-07-17 10:34:00 +00:00
return False
if messageJson['object']['actor'] != messageJson['actor']:
if debug:
print('DEBUG: actors do not match')
return False
2019-07-17 10:34:00 +00:00
nicknameFollower=getNicknameFromActor(messageJson['object']['actor'])
2019-09-02 09:43:43 +00:00
if not nicknameFollower:
print('WARN: unable to find nickname in '+messageJson['object']['actor'])
return False
2019-07-17 10:34:00 +00:00
domainFollower,portFollower=getDomainFromActor(messageJson['object']['actor'])
domainFollowerFull=domainFollower
if portFollower:
if portFollower!=80 and portFollower!=443:
if ':' not in domainFollower:
domainFollowerFull=domainFollower+':'+str(portFollower)
2019-07-17 10:34:00 +00:00
nicknameFollowing=getNicknameFromActor(messageJson['object']['object'])
2019-09-02 09:43:43 +00:00
if not nicknameFollowing:
print('WARN: unable to find nickname in '+messageJson['object']['object'])
return False
2019-07-17 10:34:00 +00:00
domainFollowing,portFollowing=getDomainFromActor(messageJson['object']['object'])
domainFollowingFull=domainFollowing
if portFollowing:
if portFollowing!=80 and portFollowing!=443:
if ':' not in domainFollowing:
domainFollowingFull=domainFollowing+':'+str(portFollowing)
2019-07-17 10:34:00 +00:00
2019-07-17 11:54:13 +00:00
if unfollowerOfPerson(baseDir, \
nicknameFollowing,domainFollowingFull, \
nicknameFollower,domainFollowerFull, \
debug):
if debug:
print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was removed')
return True
if debug:
print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was not removed')
return False
2019-07-17 10:34:00 +00:00
def receiveUndo(session,baseDir: str,httpPrefix: str, \
port: int,sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
messageJson: {},federationList: [], \
debug : bool, \
acceptedCaps=["inbox:write","objects:read"]) -> bool:
"""Receives an undo request within the POST section of HTTPServer
"""
if not messageJson['type'].startswith('Undo'):
return False
2019-07-17 11:24:11 +00:00
if debug:
print('DEBUG: Undo activity received')
2019-07-17 10:34:00 +00:00
if not messageJson.get('actor'):
if debug:
print('DEBUG: follow request has no actor')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-17 10:34:00 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor')
2019-07-17 10:34:00 +00:00
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], dict):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a dict')
return False
if not messageJson['object'].get('type'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object type')
return False
if not messageJson['object'].get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object within object')
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object within object is not a string')
return False
if messageJson['object']['type']=='Follow':
return receiveUndoFollow(session,baseDir,httpPrefix, \
2019-07-17 10:38:10 +00:00
port,messageJson, \
federationList, \
debug)
2019-07-17 10:34:00 +00:00
return False
def personReceiveUpdate(baseDir: str, \
domain: str,port: int, \
updateNickname: str,updateDomain: str,updatePort: int, \
2019-08-20 19:41:58 +00:00
personJson: {},personCache: {},debug: bool) -> bool:
"""Changes an actor. eg: avatar or display name change
2019-08-20 19:41:58 +00:00
"""
if debug:
print('DEBUG: receiving actor update for '+personJson['url'])
domainFull=domain
if port:
if port!=80 and port!=443:
domainFull=domain+':'+str(port)
2019-08-22 18:10:46 +00:00
updateDomainFull=updateDomain
if updatePort:
if updatePort!=80 and updatePort!=443:
updateDomainFull=updateDomain+':'+str(updatePort)
2019-08-22 18:07:29 +00:00
actor=updateDomainFull+'/users/'+updateNickname
if actor not in personJson['id']:
actor=updateDomainFull+'/profile/'+updateNickname
if actor not in personJson['id']:
if debug:
print('actor: '+actor)
print('id: '+personJson['id'])
print('DEBUG: Actor does not match id')
return False
if updateDomainFull==domainFull:
if debug:
print('DEBUG: You can only receive actor updates for domains other than your own')
2019-08-20 19:41:58 +00:00
return False
if not personJson.get('publicKey'):
if debug:
print('DEBUG: actor update does not contain a public key')
return False
if not personJson['publicKey'].get('publicKeyPem'):
if debug:
print('DEBUG: actor update does not contain a public key Pem')
return False
actorFilename=baseDir+'/cache/actors/'+personJson['id'].replace('/','#')+'.json'
# check that the public keys match.
# If they don't then this may be a nefarious attempt to hack an account
if personCache.get(personJson['id']):
2019-08-22 18:13:07 +00:00
if personCache[personJson['id']]['actor']['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
2019-08-20 19:41:58 +00:00
if debug:
print('WARN: Public key does not match when updating actor')
return False
else:
if os.path.isfile(actorFilename):
loadedActor=False
2019-10-11 18:03:58 +00:00
tries=0
while tries<5:
try:
with open(actorFilename, 'r') as fp:
existingPersonJson=commentjson.load(fp)
loadedActor=True
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
if loadedActor:
2019-08-20 19:41:58 +00:00
if existingPersonJson['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
if debug:
print('WARN: Public key does not match cached actor when updating')
return False
# save to cache in memory
2019-08-22 17:33:04 +00:00
storePersonInCache(baseDir,personJson['id'],personJson,personCache)
2019-08-20 19:41:58 +00:00
# save to cache on file
2019-09-30 22:39:02 +00:00
try:
with open(actorFilename, 'w') as fp:
commentjson.dump(personJson, fp, indent=4, sort_keys=False)
print('actor updated for '+personJson['id'])
except Exception as e:
print(e)
# remove avatar if it exists so that it will be refreshed later
# when a timeline is constructed
actorStr=personJson['id'].replace('/','-')
avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.png'
if os.path.isfile(avatarFilename):
os.remove(avatarFilename)
else:
avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.jpg'
if os.path.isfile(avatarFilename):
os.remove(avatarFilename)
else:
avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.gif'
if os.path.isfile(avatarFilename):
os.remove(avatarFilename)
2019-08-20 19:41:58 +00:00
return True
2019-07-09 14:20:23 +00:00
def receiveUpdate(session,baseDir: str, \
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an Update activity within the POST section of HTTPServer
"""
if messageJson['type']!='Update':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], dict):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a dict')
return False
if not messageJson['object'].get('type'):
if debug:
print('DEBUG: '+messageJson['type']+' object has no type')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-09 14:20:23 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
2019-07-09 14:20:23 +00:00
return False
2019-08-22 17:25:12 +00:00
2019-08-22 19:53:24 +00:00
if messageJson['object']['type']=='Person' or \
2019-08-23 20:09:00 +00:00
messageJson['object']['type']=='Application' or \
2019-10-04 09:23:38 +00:00
messageJson['object']['type']=='Group' or \
2019-08-22 19:53:24 +00:00
messageJson['object']['type']=='Service':
2019-08-22 17:25:12 +00:00
if messageJson['object'].get('url') and messageJson['object'].get('id'):
2019-08-22 18:07:29 +00:00
print('Request to update actor: '+messageJson['actor'])
2019-09-02 09:43:43 +00:00
updateNickname=getNicknameFromActor(messageJson['actor'])
if updateNickname:
updateDomain,updatePort=getDomainFromActor(messageJson['actor'])
if personReceiveUpdate(baseDir, \
domain,port, \
updateNickname,updateDomain,updatePort, \
messageJson['object'], \
personCache,debug):
if debug:
print('DEBUG: Profile update was received for '+messageJson['object']['url'])
return True
2019-08-22 17:25:12 +00:00
2019-07-09 14:20:23 +00:00
if messageJson['object'].get('capability') and messageJson['object'].get('scope'):
nickname=getNicknameFromActor(messageJson['object']['scope'])
2019-09-02 09:43:43 +00:00
if nickname:
domain,tempPort=getDomainFromActor(messageJson['object']['scope'])
if messageJson['object']['type']=='Capability':
if capabilitiesReceiveUpdate(baseDir,nickname,domain,port,
messageJson['actor'], \
messageJson['object']['id'], \
messageJson['object']['capability'], \
debug):
if debug:
print('DEBUG: An update was received')
return True
2019-07-09 14:20:23 +00:00
return False
2019-10-04 12:22:56 +00:00
def receiveLike(session,handle: str,isGroup: bool,baseDir: str, \
2019-07-10 12:40:31 +00:00
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives a Like activity within the POST section of HTTPServer
"""
if messageJson['type']!='Like':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-10 12:40:31 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
2019-07-10 12:40:31 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of like - '+handle)
# if this post in the outbox of the person?
2019-07-11 12:29:31 +00:00
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
2019-07-10 12:40:31 +00:00
if not postFilename:
if debug:
print('DEBUG: post not found in inbox or outbox')
print(messageJson['object'])
return True
if debug:
2019-07-11 12:59:00 +00:00
print('DEBUG: liked post found in inbox')
updateLikesCollection(postFilename,messageJson['object'],messageJson['actor'],debug)
2019-07-10 12:40:31 +00:00
return True
2019-10-04 12:22:56 +00:00
def receiveUndoLike(session,handle: str,isGroup: bool,baseDir: str, \
2019-07-12 09:10:09 +00:00
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an undo like activity within the POST section of HTTPServer
"""
if messageJson['type']!='Undo':
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('type'):
return False
if messageJson['object']['type']!='Like':
return False
if not messageJson['object'].get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' like has no object')
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' like object is not a string')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-12 09:10:09 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' like')
2019-07-12 09:10:09 +00:00
return False
if '/statuses/' not in messageJson['object']['object']:
if debug:
print('DEBUG: "statuses" missing from like object in '+messageJson['type'])
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of undo like - '+handle)
# if this post in the outbox of the person?
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object']['object'])
if not postFilename:
if debug:
2019-07-12 09:41:57 +00:00
print('DEBUG: unliked post not found in inbox or outbox')
2019-07-12 09:10:09 +00:00
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: liked post found in inbox. Now undoing.')
undoLikesCollectionEntry(postFilename,messageJson['object'],messageJson['actor'],debug)
return True
2019-10-04 12:22:56 +00:00
def receiveDelete(session,handle: str,isGroup: bool,baseDir: str, \
2019-07-11 21:38:28 +00:00
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
2019-08-12 18:02:29 +00:00
debug : bool,allowDeletion: bool) -> bool:
2019-07-11 21:38:28 +00:00
"""Receives a Delete activity within the POST section of HTTPServer
"""
if messageJson['type']!='Delete':
return False
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
2019-07-17 17:16:48 +00:00
if debug:
print('DEBUG: Delete activity arrived')
2019-07-11 21:38:28 +00:00
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
2019-08-12 18:02:29 +00:00
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
2019-08-12 18:02:29 +00:00
deletePrefix=httpPrefix+'://'+domainFull+'/'
if not allowDeletion and \
(not messageJson['object'].startswith(deletePrefix) or \
not messageJson['actor'].startswith(deletePrefix)):
if debug:
print('DEBUG: delete not permitted from other instances')
return False
2019-07-11 21:38:28 +00:00
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-11 21:38:28 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
2019-07-11 21:38:28 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
if messageJson['actor'] not in messageJson['object']:
if debug:
print('DEBUG: actor is not the owner of the post to be deleted')
2019-07-11 21:38:28 +00:00
if not os.path.isdir(baseDir+'/accounts/'+handle):
2019-08-12 18:02:29 +00:00
print('DEBUG: unknown recipient of like - '+handle)
2019-07-11 21:38:28 +00:00
# if this post in the outbox of the person?
2019-08-16 19:48:32 +00:00
messageId=messageJson['object'].replace('/activity','').replace('/undo','')
2019-08-12 18:02:29 +00:00
removeModerationPostFromIndex(baseDir,messageId,debug)
2019-07-17 17:16:48 +00:00
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageId)
2019-07-11 21:38:28 +00:00
if not postFilename:
if debug:
print('DEBUG: delete post not found in inbox or outbox')
2019-07-17 17:16:48 +00:00
print(messageId)
return True
2019-07-14 17:02:41 +00:00
deletePost(baseDir,httpPrefix,handle.split('@')[0],handle.split('@')[1],postFilename,debug)
2019-07-11 21:38:28 +00:00
if debug:
print('DEBUG: post deleted - '+postFilename)
return True
2019-10-04 12:22:56 +00:00
def receiveAnnounce(session,handle: str,isGroup: bool,baseDir: str, \
2019-07-11 19:31:02 +00:00
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
2019-07-12 09:41:57 +00:00
"""Receives an announce activity within the POST section of HTTPServer
2019-07-11 19:31:02 +00:00
"""
if messageJson['type']!='Announce':
return False
if '@' not in handle:
if debug:
print('DEBUG: bad handle '+handle)
return False
2019-07-11 19:31:02 +00:00
if not messageJson.get('actor'):
if debug:
print('DEBUG: '+messageJson['type']+' has no actor')
return False
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: receiving announce on '+handle)
2019-07-11 19:31:02 +00:00
if not messageJson.get('object'):
if debug:
print('DEBUG: '+messageJson['type']+' has no object')
return False
if not isinstance(messageJson['object'], str):
if debug:
print('DEBUG: '+messageJson['type']+' object is not a string')
return False
if not messageJson.get('to'):
if debug:
print('DEBUG: '+messageJson['type']+' has no "to" list')
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-11 19:31:02 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
return False
if '/users/' not in messageJson['object'] and '/profile/' not in messageJson['object']:
if debug:
print('DEBUG: "users" or "profile" missing in '+messageJson['type'])
2019-07-11 19:31:02 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type'])
return False
objectDomain=messageJson['object'].replace('https://','').replace('http://','').replace('dat://','')
if '/' in objectDomain:
objectDomain=objectDomain.split('/')[0]
if isBlockedDomain(baseDir,objectDomain):
if debug:
print('DEBUG: announced domain is blocked')
return False
2019-07-11 19:31:02 +00:00
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of announce - '+handle)
# is this post in the outbox of the person?
2019-09-29 09:20:01 +00:00
nickname=handle.split('@')[0]
postFilename=locatePost(baseDir,nickname,handle.split('@')[1],messageJson['object'])
2019-07-11 19:31:02 +00:00
if not postFilename:
if debug:
print('DEBUG: announce post not found in inbox or outbox')
print(messageJson['object'])
return True
updateAnnounceCollection(postFilename,messageJson['actor'],debug)
2019-09-29 10:13:00 +00:00
if debug:
2019-10-01 14:00:06 +00:00
print('DEBUG: Downloading announce post '+messageJson['actor']+' -> '+messageJson['object'])
2019-09-30 19:13:14 +00:00
postJsonObject=downloadAnnounce(session,baseDir,httpPrefix,nickname,domain,messageJson,__version__)
if postJsonObject:
2019-10-01 13:23:22 +00:00
if debug:
2019-10-01 14:00:06 +00:00
print('DEBUG: Announce post downloaded for '+messageJson['actor']+' -> '+messageJson['object'])
2019-09-30 19:13:14 +00:00
# Try to obtain the actor for this person
# so that their avatar can be shown
lookupActor=None
2019-10-01 14:11:15 +00:00
if postJsonObject.get('attributedTo'):
lookupActor=postJsonObject['attributedTo']
else:
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('attributedTo'):
lookupActor=postJsonObject['object']['attributedTo']
2019-09-30 19:13:14 +00:00
if lookupActor:
2019-10-01 14:29:28 +00:00
if '/users/' in lookupActor or '/profile/' in lookupActor:
2019-10-01 13:23:22 +00:00
if '/statuses/' in lookupActor:
lookupActor=lookupActor.split('/statuses/')[0]
2019-10-01 12:35:39 +00:00
2019-10-01 12:50:06 +00:00
if debug:
2019-10-01 13:23:22 +00:00
print('DEBUG: Obtaining actor for announce post '+lookupActor)
for tries in range(6):
pubKey= \
getPersonPubKey(baseDir,session,lookupActor, \
personCache,debug, \
__version__,httpPrefix,domain)
if pubKey:
print('DEBUG: public key obtained for announce: '+lookupActor)
break
if debug:
print('DEBUG: Retry '+str(tries+1)+ \
' obtaining actor for '+lookupActor)
time.sleep(5)
2019-07-11 19:31:02 +00:00
if debug:
2019-09-29 10:13:00 +00:00
print('DEBUG: announced/repeated post arrived in inbox')
2019-07-11 19:31:02 +00:00
return True
2019-10-04 12:22:56 +00:00
def receiveUndoAnnounce(session,handle: str,isGroup: bool,baseDir: str, \
2019-07-12 09:41:57 +00:00
httpPrefix: str,domain :str,port: int, \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},messageJson: {},federationList: [], \
debug : bool) -> bool:
"""Receives an undo announce activity within the POST section of HTTPServer
"""
if messageJson['type']!='Undo':
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('object'):
return False
if not isinstance(messageJson['object']['object'], str):
return False
if messageJson['object']['type']!='Announce':
return False
2019-09-09 09:41:31 +00:00
if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
2019-07-12 09:41:57 +00:00
if debug:
2019-09-09 09:41:31 +00:00
print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' announce')
2019-07-12 09:41:57 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
print('DEBUG: "statuses" missing from object in '+messageJson['type']+' announce')
return False
if not os.path.isdir(baseDir+'/accounts/'+handle):
print('DEBUG: unknown recipient of undo announce - '+handle)
# if this post in the outbox of the person?
postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
if not postFilename:
if debug:
print('DEBUG: undo announce post not found in inbox or outbox')
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: announced/repeated post to be undone found in inbox')
loadedPost=False
2019-10-11 18:03:58 +00:00
tries=0
while tries<5:
try:
with open(postFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
loadedPost=True
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
if loadedPost:
2019-07-14 16:57:06 +00:00
if not postJsonObject.get('type'):
if postJsonObject['type']!='Announce':
if debug:
print("DEBUG: Attempt to undo something which isn't an announcement")
return False
undoAnnounceCollectionEntry(postFilename,messageJson['actor'],debug)
2019-07-12 09:41:57 +00:00
os.remove(postFilename)
return True
def populateReplies(baseDir :str,httpPrefix :str,domain :str, \
2019-07-13 21:00:12 +00:00
messageJson :{},maxReplies: int,debug :bool) -> bool:
"""Updates the list of replies for a post on this domain if
a reply to it arrives
"""
if not messageJson.get('id'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('inReplyTo'):
return False
if not messageJson['object'].get('to'):
return False
replyTo=messageJson['object']['inReplyTo']
if debug:
print('DEBUG: post contains a reply')
# is this a reply to a post on this domain?
if not replyTo.startswith(httpPrefix+'://'+domain+'/'):
if debug:
print('DEBUG: post is a reply to another not on this domain')
2019-08-02 18:04:31 +00:00
print(replyTo)
print('Expected: '+httpPrefix+'://'+domain+'/')
return False
replyToNickname=getNicknameFromActor(replyTo)
if not replyToNickname:
2019-09-02 09:43:43 +00:00
print('DEBUG: no nickname found for '+replyTo)
return False
replyToDomain,replyToPort=getDomainFromActor(replyTo)
if not replyToDomain:
if debug:
print('DEBUG: no domain found for '+replyTo)
return False
postFilename=locatePost(baseDir,replyToNickname,replyToDomain,replyTo)
if not postFilename:
if debug:
print('DEBUG: post may have expired - '+replyTo)
2019-07-13 19:28:14 +00:00
return False
# populate a text file containing the ids of replies
postRepliesFilename=postFilename.replace('.json','.replies')
2019-08-16 19:48:32 +00:00
messageId=messageJson['id'].replace('/activity','').replace('/undo','')
2019-07-13 19:28:14 +00:00
if os.path.isfile(postRepliesFilename):
2019-07-13 21:00:12 +00:00
numLines = sum(1 for line in open(postRepliesFilename))
2019-08-02 18:04:31 +00:00
if numLines>maxReplies:
2019-07-13 21:00:12 +00:00
return False
2019-07-13 19:28:14 +00:00
if messageId not in open(postRepliesFilename).read():
repliesFile=open(postRepliesFilename, "a")
repliesFile.write(messageId+'\n')
repliesFile.close()
else:
repliesFile=open(postRepliesFilename, "w")
repliesFile.write(messageId+'\n')
repliesFile.close()
return True
2019-09-30 09:43:46 +00:00
2019-09-30 10:15:20 +00:00
def estimateNumberOfMentions(content: str) -> int:
"""Returns a rough estimate of the number of mentions
"""
words=content.split(' ')
ctr=0
for word in words:
if word.startswith('@') or '>@' in word:
ctr+=1
return ctr
def validPostContent(messageJson: {},maxMentions: int) -> bool:
2019-09-30 09:43:46 +00:00
"""Is the content of a received post valid?
2019-09-30 10:15:20 +00:00
Check for bad html
Check for hellthreads
Check number of tags is reasonable
2019-09-30 09:43:46 +00:00
"""
if not messageJson.get('object'):
return True
if not isinstance(messageJson['object'], dict):
return True
if not messageJson['object'].get('content'):
return True
# check for bad html
2019-09-30 11:12:02 +00:00
invalidStrings=['<script>','<canvas>','<style>','</html>','</body>','<br>','<hr>']
2019-09-30 09:43:46 +00:00
for badStr in invalidStrings:
if badStr in messageJson['object']['content']:
if messageJson['object'].get('id'):
print('REJECT: '+messageJson['object']['id'])
2019-09-30 09:53:43 +00:00
print('REJECT: bad string in post - '+messageJson['object']['content'])
2019-09-30 09:43:46 +00:00
return False
# check (rough) number of mentions
2019-09-30 10:15:20 +00:00
if estimateNumberOfMentions(messageJson['object']['content'])>maxMentions:
2019-09-30 10:37:34 +00:00
if messageJson['object'].get('id'):
print('REJECT: '+messageJson['object']['id'])
2019-09-30 10:15:20 +00:00
print('REJECT: Too many mentions in post - '+messageJson['object']['content'])
return False
# check number of tags
if messageJson['object'].get('tag'):
if not isinstance(messageJson['object']['tag'], list):
messageJson['object']['tag']=[]
else:
if len(messageJson['object']['tag']) > maxMentions*2:
2019-09-30 10:37:34 +00:00
if messageJson['object'].get('id'):
print('REJECT: '+messageJson['object']['id'])
print('REJECT: Too many tags in post - '+messageJson['object']['tag'])
return False
2019-09-30 09:43:46 +00:00
print('ACCEPT: post content is valid')
return True
def obtainAvatarForReplyPost(session,baseDir: str,httpPrefix: str,domain: str,personCache: {},postJsonObject: {},debug: bool) -> None:
"""Tries to obtain the actor for the person being replied to
so that their avatar can later be shown
"""
2019-09-30 19:39:48 +00:00
if not postJsonObject.get('object'):
return
if not isinstance(postJsonObject['object'], dict):
return
if not postJsonObject['object'].get('inReplyTo'):
return
lookupActor=postJsonObject['object']['inReplyTo']
if lookupActor:
2019-10-01 14:29:28 +00:00
if '/users/' in lookupActor or '/profile/' in lookupActor:
2019-10-01 13:23:22 +00:00
if '/statuses/' in lookupActor:
lookupActor=lookupActor.split('/statuses/')[0]
2019-10-01 12:50:06 +00:00
if debug:
2019-10-01 13:23:22 +00:00
print('DEBUG: Obtaining actor for reply post '+lookupActor)
for tries in range(6):
pubKey= \
getPersonPubKey(baseDir,session,lookupActor, \
personCache,debug, \
__version__,httpPrefix,domain)
if pubKey:
print('DEBUG: public key obtained for reply: '+lookupActor)
break
if debug:
print('DEBUG: Retry '+str(tries+1)+ \
' obtaining actor for '+lookupActor)
time.sleep(5)
2019-10-06 15:07:40 +00:00
def dmNotify(baseDir: str,handle: str,url: str) -> None:
"""Creates a notification that a new DM has arrived
"""
accountDir=baseDir+'/accounts/'+handle
if not os.path.isdir(accountDir):
return
dmFile=accountDir+'/.newDM'
if not os.path.isfile(dmFile):
with open(dmFile, 'w') as fp:
2019-10-06 15:07:40 +00:00
fp.write(url)
2019-10-06 15:11:10 +00:00
def replyNotify(baseDir: str,handle: str,url: str) -> None:
"""Creates a notification that a new reply has arrived
"""
accountDir=baseDir+'/accounts/'+handle
if not os.path.isdir(accountDir):
return
replyFile=accountDir+'/.newReply'
if not os.path.isfile(replyFile):
with open(replyFile, 'w') as fp:
2019-10-06 15:11:10 +00:00
fp.write(url)
2019-10-04 12:22:56 +00:00
def groupHandle(baseDir: str,handle: str) -> bool:
"""Is the given account handle a group?
"""
actorFile=baseDir+'/accounts/'+handle+'.json'
if not os.path.isfile(actorFile):
return False
actorJson=None
2019-10-11 18:03:58 +00:00
tries=0
while tries<5:
try:
with open(actorFile, 'r') as fp:
actorJson=commentjson.load(fp)
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
2019-10-04 12:22:56 +00:00
if not actorJson:
return False
return actorJson['type']=='Group'
2019-10-04 13:39:41 +00:00
def getGroupName(baseDir: str,handle: str) -> str:
"""Returns the preferred name of a group
"""
actorFile=baseDir+'/accounts/'+handle+'.json'
if not os.path.isfile(actorFile):
return False
actorJson=None
2019-10-11 18:03:58 +00:00
tries=0
while tries<5:
try:
with open(actorFile, 'r') as fp:
actorJson=commentjson.load(fp)
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
2019-10-04 13:39:41 +00:00
if not actorJson:
return 'Group'
return actorJson['name']
2019-10-04 12:22:56 +00:00
def sendToGroupMembers(session,baseDir: str,handle: str,port: int,postJsonObject: {}, \
httpPrefix: str,federationList: [], \
sendThreads: [],postLog: [],cachedWebfingers: {}, \
personCache: {},debug: bool) -> None:
"""When a post arrives for a group send it out to the group members
"""
followersFile=baseDir+'/accounts/'+handle+'/followers.txt'
if not os.path.isfile(followersFile):
return
2019-10-04 13:31:30 +00:00
if not postJsonObject.get('object'):
return
2019-10-04 12:22:56 +00:00
nickname=handle.split('@')[0]
2019-10-04 13:39:41 +00:00
groupname=getGroupName(baseDir,handle)
2019-10-04 12:22:56 +00:00
domain=handle.split('@')[1]
2019-10-04 14:43:46 +00:00
domainFull=domain
2019-10-04 14:02:11 +00:00
if ':' not in domain:
if port:
if port!=80 and port !=443:
domain=domain+':'+str(port)
# set sender
2019-10-04 14:18:04 +00:00
cc=''
sendingActor=postJsonObject['actor']
sendingActorNickname=getNicknameFromActor(sendingActor)
sendingActorDomain,sendingActorPort=getDomainFromActor(sendingActor)
sendingActorDomainFull=sendingActorDomain
if ':' in sendingActorDomain:
if sendingActorPort:
if sendingActorPort!=80 and sendingActorPort!=443:
sendingActorDomainFull=sendingActorDomain+':'+str(sendingActorPort)
senderStr='@'+sendingActorNickname+'@'+sendingActorDomainFull
if not postJsonObject['object']['content'].startswith(senderStr):
postJsonObject['object']['content']=senderStr+' '+postJsonObject['object']['content']
# add mention to tag list
2019-10-04 14:38:18 +00:00
if not postJsonObject['object']['tag']:
postJsonObject['object']['tag']=[]
2019-10-04 15:17:48 +00:00
# check if the mention already exists
mentionExists=False
for mention in postJsonObject['object']['tag']:
if mention['type']=='Mention':
if mention.get('href'):
if mention['href']==sendingActor:
mentionExists=True
if not mentionExists:
# add the mention of the original sender
postJsonObject['object']['tag'].append({
'href': sendingActor,
'name': senderStr,
'type': 'Mention'
})
2019-10-04 14:43:46 +00:00
postJsonObject['actor']=httpPrefix+'://'+domainFull+'/users/'+nickname
postJsonObject['to']=[httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers']
2019-10-04 14:15:46 +00:00
postJsonObject['cc']=[cc]
2019-10-04 14:02:11 +00:00
postJsonObject['object']['to']=postJsonObject['to']
2019-10-04 14:15:46 +00:00
postJsonObject['object']['cc']=[cc]
2019-10-04 14:09:48 +00:00
# set subject
if not postJsonObject['object'].get('summary'):
postJsonObject['object']['summary']='General Discussion'
2019-10-04 12:22:56 +00:00
if ':' in domain:
domain=domain.split(':')[0]
with open(followersFile, 'r') as groupMembers:
for memberHandle in groupMembers:
if memberHandle!=handle:
memberNickname=memberHandle.split('@')[0]
2019-10-04 13:31:30 +00:00
memberDomain=memberHandle.split('@')[1]
2019-10-04 12:22:56 +00:00
memberPort=port
if ':' in memberDomain:
memberPortStr=memberDomain.split(':')[1]
if memberPortStr.isdigit():
memberPort=int(memberPortStr)
memberDomain=memberDomain.split(':')[0]
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
2019-10-04 14:15:46 +00:00
memberNickname,memberDomain,memberPort,cc, \
2019-10-04 12:22:56 +00:00
httpPrefix,False,False,federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug,projectVersion)
2019-10-11 12:31:06 +00:00
def inboxUpdateCalendar(baseDir: str,handle: str,postJsonObject: {}) -> None:
"""Detects whether the tag list on a post contains calendar events
and if so saves the post id to a file in the calendar directory
for the account
"""
if not postJsonObject.get('object'):
return
if not isinstance(postJsonObject['object'], dict):
return
if not postJsonObject['object'].get('tag'):
return
if not isinstance(postJsonObject['object']['tag'], list):
return
2019-10-11 12:33:40 +00:00
calendarPath=baseDir+'/accounts/'+handle+'/calendar'
2019-10-11 12:31:06 +00:00
if not os.path.isdir(calendarPath):
os.mkdir(calendarPath)
for tagDict in postJsonObject['object']['tag']:
if tagDict['type']!='Event':
continue
2019-10-11 16:16:56 +00:00
if not tagDict.get('startTime'):
2019-10-11 12:31:06 +00:00
continue
# get the year and month from the event
2019-10-11 16:54:55 +00:00
eventTime=datetime.datetime.strptime(tagDict['startTime'],"%Y-%m-%dT%H:%M:%S%z")
2019-10-11 12:31:06 +00:00
eventYear=int(eventTime.strftime("%Y"))
eventMonthNumber=int(eventTime.strftime("%m"))
if not os.path.isdir(calendarPath+'/'+str(eventYear)):
os.mkdir(calendarPath+'/'+str(eventYear))
calendarFilename=calendarPath+'/'+str(eventYear)+'/'+str(eventMonthNumber)+'.txt'
2019-10-11 18:08:47 +00:00
postId=postJsonObject['id'].replace('/activity','').replace('/','#')
if os.path.isfile(calendarFilename):
if postId in open(calendarFilename).read():
return
2019-10-11 12:31:06 +00:00
calendarFile=open(calendarFilename,'a+')
if calendarFile:
calendarFile.write(postId+'\n')
2019-10-11 12:31:06 +00:00
calendarFile.close()
2019-07-10 12:40:31 +00:00
def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
baseDir: str,httpPrefix: str,sendThreads: [], \
postLog: [],cachedWebfingers: {},personCache: {}, \
queue: [],domain: str,port: int,useTor: bool, \
federationList: [],ocapAlways: bool,debug: bool, \
2019-09-30 10:15:20 +00:00
acceptedCaps: [], \
queueFilename :str,destinationFilename :str, \
maxReplies: int,allowDeletion: bool, \
maxMentions: int) -> bool:
""" Anything which needs to be done after capabilities checks have passed
"""
2019-09-29 10:41:21 +00:00
actor=keyId
if '#' in actor:
actor=keyId.split('#')[0]
2019-10-04 12:22:56 +00:00
isGroup=groupHandle(baseDir,handle)
if receiveLike(session,handle,isGroup, \
2019-07-10 12:40:31 +00:00
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
2019-09-29 10:41:21 +00:00
print('DEBUG: Like accepted from '+actor)
2019-07-10 12:40:31 +00:00
return False
2019-10-04 12:22:56 +00:00
if receiveUndoLike(session,handle,isGroup, \
2019-07-12 09:10:09 +00:00
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
2019-09-29 10:41:21 +00:00
print('DEBUG: Undo like accepted from '+actor)
2019-07-12 09:10:09 +00:00
return False
2019-10-04 12:22:56 +00:00
if receiveAnnounce(session,handle,isGroup, \
2019-07-11 19:31:02 +00:00
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
2019-09-29 10:41:21 +00:00
print('DEBUG: Announce accepted from '+actor)
2019-07-11 19:31:02 +00:00
2019-10-04 12:22:56 +00:00
if receiveUndoAnnounce(session,handle,isGroup, \
2019-07-12 09:41:57 +00:00
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug):
if debug:
2019-09-29 10:41:21 +00:00
print('DEBUG: Undo announce accepted from '+actor)
2019-07-12 11:35:03 +00:00
return False
2019-07-12 09:41:57 +00:00
2019-10-04 12:22:56 +00:00
if receiveDelete(session,handle,isGroup, \
2019-08-12 18:02:29 +00:00
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
cachedWebfingers, \
personCache, \
messageJson, \
federationList, \
debug,allowDeletion):
if debug:
2019-09-29 10:41:21 +00:00
print('DEBUG: Delete accepted from '+actor)
2019-08-12 18:02:29 +00:00
return False
2019-07-10 13:32:47 +00:00
if debug:
print('DEBUG: object capabilities passed')
2019-08-17 12:26:09 +00:00
print('copy queue file from '+queueFilename+' to '+destinationFilename)
2019-08-16 22:04:45 +00:00
if os.path.isfile(destinationFilename):
return True
2019-10-04 09:58:02 +00:00
if messageJson.get('postNickname'):
2019-10-04 12:22:56 +00:00
postJsonObject=messageJson['post']
else:
2019-10-04 12:22:56 +00:00
postJsonObject=messageJson
if validPostContent(postJsonObject,maxMentions):
2019-10-11 12:00:18 +00:00
populateReplies(baseDir,httpPrefix,domain,messageJson,maxReplies,debug)
2019-10-04 12:22:56 +00:00
if not isGroup:
# create a DM notification file if needed
if isDM(postJsonObject):
2019-10-06 15:07:40 +00:00
nickname=handle.split('@')[0]
2019-10-06 15:11:10 +00:00
if nickname!='inbox':
dmNotify(baseDir,handle,httpPrefix+'://'+domain+'/users/'+nickname+'/dm')
2019-10-04 12:22:56 +00:00
# get the actor being replied to
domainFull=domain
if port:
if ':' not in domain:
if port!=80 and port!=443:
domainFull=domainFull+':'+str(port)
actor=httpPrefix+'://'+domainFull+'/users/'+handle.split('@')[0]
# create a reply notification file if needed
if isReply(postJsonObject,actor):
2019-10-06 15:11:10 +00:00
nickname=handle.split('@')[0]
if nickname!='inbox':
replyNotify(baseDir,handle,httpPrefix+'://'+domain+'/users/'+nickname+'/tlreplies')
2019-10-04 10:00:57 +00:00
# get the avatar for a reply/announce
2019-10-04 12:22:56 +00:00
obtainAvatarForReplyPost(session,baseDir,httpPrefix,domain,personCache,postJsonObject,debug)
2019-10-04 10:00:57 +00:00
# save the post to file
2019-10-04 09:58:02 +00:00
try:
with open(destinationFilename, 'w+') as fp:
2019-10-04 12:22:56 +00:00
commentjson.dump(postJsonObject, fp, indent=4, sort_keys=False)
2019-10-04 09:58:02 +00:00
except Exception as e:
print(e)
2019-08-17 12:26:09 +00:00
2019-10-11 12:31:06 +00:00
inboxUpdateCalendar(baseDir,handle,postJsonObject)
2019-10-04 12:22:56 +00:00
# send the post out to group members
if isGroup:
sendToGroupMembers(session,baseDir,handle,port,postJsonObject, \
httpPrefix,federationList,sendThreads, \
postLog,cachedWebfingers,personCache,debug)
2019-10-04 10:00:57 +00:00
# if the post wasn't saved
2019-08-17 12:26:09 +00:00
if not os.path.isfile(destinationFilename):
return False
return True
2019-07-12 21:09:23 +00:00
def restoreQueueItems(baseDir: str,queue: []) -> None:
"""Checks the queue for each account and appends filenames
"""
2019-08-15 16:45:07 +00:00
queue.clear()
2019-07-12 21:09:23 +00:00
for subdir,dirs,files in os.walk(baseDir+'/accounts'):
for account in dirs:
queueDir=baseDir+'/accounts/'+account+'/queue'
if os.path.isdir(queueDir):
for queuesubdir,queuedirs,queuefiles in os.walk(queueDir):
for qfile in queuefiles:
queue.append(os.path.join(queueDir, qfile))
2019-08-15 16:19:07 +00:00
if len(queue)>0:
2019-08-15 16:19:57 +00:00
print('Restored '+str(len(queue))+' inbox queue items')
2019-09-02 21:52:43 +00:00
def runInboxQueueWatchdog(projectVersion: str,httpd) -> None:
"""This tries to keep the inbox thread running even if it dies
"""
print('Starting inbox queue watchdog')
2019-09-03 11:10:53 +00:00
inboxQueueOriginal=httpd.thrInboxQueue.clone(runInboxQueue)
#httpd.thrInboxQueue=inboxQueueOriginal
2019-09-02 21:52:43 +00:00
httpd.thrInboxQueue.start()
while True:
time.sleep(20)
if not httpd.thrInboxQueue.isAlive():
httpd.thrInboxQueue.kill()
2019-09-03 11:10:53 +00:00
httpd.thrInboxQueue=inboxQueueOriginal.clone(runInboxQueue)
2019-09-02 21:52:43 +00:00
httpd.thrInboxQueue.start()
print('Restarting inbox queue...')
2019-08-14 20:12:27 +00:00
def runInboxQueue(projectVersion: str, \
baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \
2019-07-11 21:38:28 +00:00
cachedWebfingers: {},personCache: {},queue: [], \
domain: str,port: int,useTor: bool,federationList: [], \
2019-07-15 10:22:19 +00:00
ocapAlways: bool,maxReplies: int, \
domainMaxPostsPerDay: int,accountMaxPostsPerDay: int, \
2019-09-30 10:15:20 +00:00
allowDeletion: bool,debug: bool,maxMentions: int, \
2019-07-11 21:38:28 +00:00
acceptedCaps=["inbox:write","objects:read"]) -> None:
2019-07-04 12:23:53 +00:00
"""Processes received items and moves them to
the appropriate directories
"""
currSessionTime=int(time.time())
sessionLastUpdate=currSessionTime
session=createSession(domain,port,useTor)
2019-07-08 23:05:48 +00:00
inboxHandle='inbox@'+domain
2019-07-04 12:23:53 +00:00
if debug:
print('DEBUG: Inbox queue running')
2019-07-12 21:09:23 +00:00
# if queue processing was interrupted (eg server crash)
# then this loads any outstanding items back into the queue
restoreQueueItems(baseDir,queue)
2019-07-15 10:22:19 +00:00
# keep track of numbers of incoming posts per unit of time
quotasLastUpdate=int(time.time())
quotas={
'domains': {},
'accounts': {}
}
# keep track of the number of queue item read failures
# so that if a file is corrupt then it will eventually
# be ignored rather than endlessly retried
itemReadFailed=0
2019-09-03 08:46:26 +00:00
heartBeatCtr=0
2019-09-03 09:11:33 +00:00
queueRestoreCtr=0
2019-09-03 08:46:26 +00:00
2019-07-04 12:23:53 +00:00
while True:
2019-09-03 08:46:26 +00:00
time.sleep(1)
# heartbeat to monitor whether the inbox queue is running
heartBeatCtr+=1
if heartBeatCtr>=10:
2019-09-03 09:01:19 +00:00
print('>>> Heartbeat Q:'+str(len(queue))+' '+datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S"))
2019-09-03 08:46:26 +00:00
heartBeatCtr=0
2019-09-03 09:11:33 +00:00
if len(queue)==0:
# restore any remaining queue items
queueRestoreCtr+=1
if queueRestoreCtr>=30:
queueRestoreCtr=0
restoreQueueItems(baseDir,queue)
else:
2019-07-15 10:22:19 +00:00
currTime=int(time.time())
# recreate the session periodically
2019-08-15 16:23:38 +00:00
if not session or currTime-sessionLastUpdate>1200:
print('Creating inbox session')
2019-07-04 12:23:53 +00:00
session=createSession(domain,port,useTor)
2019-07-15 10:22:19 +00:00
sessionLastUpdate=currTime
2019-07-04 12:23:53 +00:00
# oldest item first
queue.sort()
queueFilename=queue[0]
if not os.path.isfile(queueFilename):
if debug:
print("DEBUG: queue item rejected because it has no file: "+queueFilename)
2019-07-04 12:23:53 +00:00
queue.pop(0)
continue
2019-08-15 16:36:39 +00:00
print('Loading queue item '+queueFilename)
2019-07-04 12:23:53 +00:00
# Load the queue json
try:
with open(queueFilename, 'r') as fp:
queueJson=commentjson.load(fp)
except:
itemReadFailed+=1
print('WARN: Failed to load inbox queue item '+queueFilename+' (try '+str(itemReadFailed)+')')
if itemReadFailed>4:
# After a few tries we can assume that the file
# is probably corrupt/unreadable
queue.pop(0)
itemReadFailed=0
2019-09-03 08:46:26 +00:00
# delete the queue file
os.remove(queueFilename)
continue
itemReadFailed=0
2019-07-15 10:22:19 +00:00
# clear the daily quotas for maximum numbers of received posts
if currTime-quotasLastUpdate>60*60*24:
quotas={
'domains': {},
'accounts': {}
}
quotasLastUpdate=currTime
# limit the number of posts which can arrive per domain per day
postDomain=queueJson['postDomain']
if postDomain:
2019-07-15 10:25:13 +00:00
if domainMaxPostsPerDay>0:
if quotas['domains'].get(postDomain):
if quotas['domains'][postDomain]>domainMaxPostsPerDay:
if debug:
print('DEBUG: Maximum posts for '+postDomain+' reached')
2019-07-15 10:25:13 +00:00
queue.pop(0)
continue
quotas['domains'][postDomain]+=1
else:
quotas['domains'][postDomain]=1
if accountMaxPostsPerDay>0:
postHandle=queueJson['postNickname']+'@'+postDomain
if quotas['accounts'].get(postHandle):
if quotas['accounts'][postHandle]>accountMaxPostsPerDay:
if debug:
print('DEBUG: Maximum posts for '+postHandle+' reached')
2019-07-15 10:25:13 +00:00
queue.pop(0)
continue
quotas['accounts'][postHandle]+=1
else:
quotas['accounts'][postHandle]=1
2019-07-15 10:22:19 +00:00
if debug:
2019-07-15 10:25:13 +00:00
if accountMaxPostsPerDay>0 or domainMaxPostsPerDay>0:
pprint(quotas)
2019-07-15 10:22:19 +00:00
2019-08-16 09:35:06 +00:00
print('Obtaining public key for actor '+queueJson['actor'])
2019-08-15 16:19:07 +00:00
2019-07-04 19:34:28 +00:00
# Try a few times to obtain the public key
2019-07-04 12:23:53 +00:00
pubKey=None
keyId=None
2019-07-04 17:31:41 +00:00
for tries in range(8):
2019-07-04 14:36:29 +00:00
keyId=None
2019-08-16 08:43:53 +00:00
signatureParams=queueJson['httpHeaders']['signature'].split(',')
2019-07-04 14:36:29 +00:00
for signatureItem in signatureParams:
if signatureItem.startswith('keyId='):
if '"' in signatureItem:
keyId=signatureItem.split('"')[1]
break
if not keyId:
if debug:
2019-08-05 10:14:23 +00:00
print('DEBUG: No keyId in signature: '+ \
2019-08-16 08:43:53 +00:00
queueJson['httpHeaders']['signature'])
2019-07-04 14:36:29 +00:00
os.remove(queueFilename)
queue.pop(0)
continue
2019-08-14 20:12:27 +00:00
pubKey= \
2019-08-20 09:16:03 +00:00
getPersonPubKey(baseDir,session,keyId, \
personCache,debug, \
2019-08-14 20:12:27 +00:00
projectVersion,httpPrefix,domain)
2019-07-04 17:31:41 +00:00
if pubKey:
print('DEBUG: public key: '+str(pubKey))
break
if debug:
2019-08-05 10:14:23 +00:00
print('DEBUG: Retry '+str(tries+1)+ \
' obtaining public key for '+keyId)
2019-07-04 17:31:41 +00:00
time.sleep(5)
2019-08-05 09:50:45 +00:00
if not pubKey:
if debug:
print('DEBUG: public key could not be obtained from '+keyId)
2019-07-28 13:30:19 +00:00
os.remove(queueFilename)
queue.pop(0)
2019-08-05 09:50:45 +00:00
continue
2019-07-04 12:23:53 +00:00
2019-08-05 09:50:45 +00:00
# check the signature
2019-08-15 08:36:49 +00:00
if debug:
print('DEBUG: checking http headers')
2019-08-16 08:44:56 +00:00
pprint(queueJson['httpHeaders'])
2019-08-05 09:50:45 +00:00
if not verifyPostHeaders(httpPrefix, \
2019-08-15 22:12:58 +00:00
pubKey, \
queueJson['httpHeaders'], \
2019-08-05 10:14:23 +00:00
queueJson['path'],False, \
queueJson['digest'], \
2019-08-05 09:50:45 +00:00
json.dumps(queueJson['post'])):
if debug:
print('DEBUG: Header signature check failed')
2019-07-28 13:30:19 +00:00
os.remove(queueFilename)
queue.pop(0)
2019-08-05 09:50:45 +00:00
continue
2019-07-04 12:23:53 +00:00
2019-08-05 09:50:45 +00:00
if debug:
print('DEBUG: Signature check success')
2019-07-04 17:31:41 +00:00
2019-08-16 15:04:40 +00:00
# set the id to the same as the post filename
# This makes the filename and the id consistent
#if queueJson['post'].get('id'):
# queueJson['post']['id']=queueJson['id']
2019-07-17 10:34:00 +00:00
if receiveUndo(session, \
baseDir,httpPrefix,port, \
sendThreads,postLog, \
cachedWebfingers,
2019-08-05 10:14:23 +00:00
personCache, \
2019-07-17 10:34:00 +00:00
queueJson['post'], \
federationList, \
debug, \
acceptedCaps=["inbox:write","objects:read"]):
if debug:
print('DEBUG: Undo accepted from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
2019-08-15 16:05:28 +00:00
if debug:
print('DEBUG: checking for follow requests')
2019-07-05 18:57:19 +00:00
if receiveFollowRequest(session, \
baseDir,httpPrefix,port, \
sendThreads,postLog, \
cachedWebfingers,
2019-08-05 10:14:23 +00:00
personCache, \
2019-07-04 20:25:19 +00:00
queueJson['post'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-08-14 20:12:27 +00:00
debug,projectVersion, \
2019-07-09 17:54:08 +00:00
acceptedCaps=["inbox:write","objects:read"]):
2019-07-05 18:57:19 +00:00
os.remove(queueFilename)
queue.pop(0)
2019-08-31 15:17:07 +00:00
if debug:
print('DEBUG: Follow activity for '+keyId+' removed from accepted from queue')
2019-07-05 18:57:19 +00:00
continue
2019-08-15 16:05:28 +00:00
else:
if debug:
print('DEBUG: No follow requests')
2019-07-06 15:17:21 +00:00
if receiveAcceptReject(session, \
2019-07-06 19:24:52 +00:00
baseDir,httpPrefix,domain,port, \
2019-07-06 15:17:21 +00:00
sendThreads,postLog, \
2019-08-05 10:14:23 +00:00
cachedWebfingers, \
personCache, \
2019-07-06 15:17:21 +00:00
queueJson['post'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-06 15:17:21 +00:00
debug):
if debug:
print('DEBUG: Accept/Reject received from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
2019-07-09 14:20:23 +00:00
if receiveUpdate(session, \
baseDir,httpPrefix, \
domain,port, \
sendThreads,postLog, \
2019-08-05 10:14:23 +00:00
cachedWebfingers, \
personCache, \
2019-07-09 14:20:23 +00:00
queueJson['post'], \
federationList, \
debug):
if debug:
print('DEBUG: Update accepted from '+keyId)
os.remove(queueFilename)
queue.pop(0)
continue
# get recipients list
recipientsDict,recipientsDictFollowers= \
2019-08-05 10:14:23 +00:00
inboxPostRecipients(baseDir,queueJson['post'], \
httpPrefix,domain,port,debug)
2019-07-11 12:29:31 +00:00
if len(recipientsDict.items())==0 and \
len(recipientsDictFollowers.items())==0:
if debug:
pprint(queueJson['post'])
print('DEBUG: no recipients were resolved for post arriving in inbox')
os.remove(queueFilename)
queue.pop(0)
continue
# if there are only a small number of followers then process them as if they
# were specifically addresses to particular accounts
noOfFollowItems=len(recipientsDictFollowers.items())
if noOfFollowItems>0:
if noOfFollowItems<5:
if debug:
2019-08-05 10:14:23 +00:00
print('DEBUG: moving '+str(noOfFollowItems)+ \
' inbox posts addressed to followers')
for handle,postItem in recipientsDictFollowers.items():
2019-07-11 12:29:31 +00:00
recipientsDict[handle]=postItem
recipientsDictFollowers={}
recipientsList=[recipientsDict,recipientsDictFollowers]
if debug:
print('*************************************')
print('Resolved recipients list:')
pprint(recipientsDict)
2019-07-11 12:29:31 +00:00
print('Resolved followers list:')
pprint(recipientsDictFollowers)
print('*************************************')
2019-07-08 23:05:48 +00:00
if queueJson['post'].get('capability'):
if not isinstance(queueJson['post']['capability'], list):
if debug:
2019-07-08 23:05:48 +00:00
print('DEBUG: capability on post should be a list')
os.remove(queueFilename)
queue.pop(0)
continue
# Copy any posts addressed to followers into the shared inbox
# this avoid copying file multiple times to potentially many
# individual inboxes
# This obviously bypasses object capabilities and so
# any checking will needs to be handled at the time when inbox
# GET happens on individual accounts.
# See posts.py/createBoxBase
if len(recipientsDictFollowers)>0:
sharedInboxPostFilename=queueJson['destination'].replace(inboxHandle,inboxHandle)
if not os.path.isfile(sharedInboxPostFilename):
2019-09-30 22:39:02 +00:00
try:
with open(sharedInboxPostFilename, 'w') as fp:
commentjson.dump(queueJson['post'],fp,indent=4, \
sort_keys=False)
except Exception as e:
print(e)
# for posts addressed to specific accounts
for handle,capsId in recipientsDict.items():
destination=queueJson['destination'].replace(inboxHandle,handle)
# check that capabilities are accepted
2019-07-08 23:05:48 +00:00
if queueJson['post'].get('capability'):
capabilityIdList=queueJson['post']['capability']
# does the capability id list within the post contain the id
# of the recipient with this handle?
# Here the capability id begins with the handle, so this could also
# be matched separately, but it's probably not necessary
2019-07-08 23:05:48 +00:00
if capsId in capabilityIdList:
inboxAfterCapabilities(session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \
cachedWebfingers, \
personCache,queue,domain, \
port,useTor, \
federationList,ocapAlways, \
debug,acceptedCaps, \
2019-07-13 21:00:12 +00:00
queueFilename,destination, \
2019-09-30 10:15:20 +00:00
maxReplies,allowDeletion, \
maxMentions)
2019-07-08 23:05:48 +00:00
else:
if debug:
2019-08-18 09:39:12 +00:00
print('DEBUG: object capabilities check has failed')
2019-07-08 23:05:48 +00:00
pprint(queueJson['post'])
else:
if not ocapAlways:
inboxAfterCapabilities(session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \
cachedWebfingers, \
personCache,queue,domain, \
port,useTor, \
federationList,ocapAlways, \
debug,acceptedCaps, \
2019-07-13 21:00:12 +00:00
queueFilename,destination, \
2019-09-30 10:15:20 +00:00
maxReplies,allowDeletion, \
maxMentions)
2019-07-09 08:44:24 +00:00
if debug:
2019-08-18 09:39:12 +00:00
pprint(queueJson['post'])
print('No capability list within post')
print('ocapAlways: '+str(ocapAlways))
2019-07-09 08:44:24 +00:00
print('DEBUG: object capabilities check failed')
2019-07-08 23:05:48 +00:00
if debug:
print('DEBUG: Queue post accepted')
2019-07-08 23:05:48 +00:00
os.remove(queueFilename)
2019-07-04 12:23:53 +00:00
queue.pop(0)