epicyon/inbox.py

2996 lines
118 KiB
Python
Raw Normal View History

2020-04-03 16:27:34 +00:00
__filename__ = "inbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.1.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
2019-06-28 21:59:54 +00:00
import json
import os
2019-06-29 10:08:59 +00:00
import datetime
2019-07-04 12:23:53 +00:00
import time
2020-08-26 17:41:38 +00:00
from utils import isEventPost
2020-08-23 11:13:35 +00:00
from utils import removeIdEnding
2020-06-11 12:26:15 +00:00
from utils import getProtocolPrefixes
2020-02-25 15:24:29 +00:00
from utils import isBlogPost
from utils import removeAvatarFromCache
2019-12-12 17:34:31 +00:00
from utils import isPublicPost
from utils import getCachedPostFilename
from utils import removePostFromCache
2019-07-02 10:39:55 +00:00
from utils import urlPermitted
2019-07-04 10:02:56 +00:00
from utils import createInboxQueueDir
2019-07-06 13:49:25 +00:00
from utils import getStatusNumber
2019-07-09 14:20:23 +00:00
from utils import getDomainFromActor
from utils import getNicknameFromActor
2019-07-11 12:29:31 +00:00
from utils import locatePost
2019-07-14 16:37:01 +00:00
from utils import deletePost
2019-08-12 18:02:29 +00:00
from utils import removeModerationPostFromIndex
2019-10-22 11:55:06 +00:00
from utils import loadJson
from utils import saveJson
2019-07-04 12:23:53 +00:00
from httpsig import verifyPostHeaders
from session import createSession
2019-07-04 19:34:28 +00:00
from session import getJson
2019-07-04 12:23:53 +00:00
from follow import receiveFollowRequest
2019-07-08 18:55:39 +00:00
from follow import getFollowersOfActor
2019-07-17 11:54:13 +00:00
from follow import unfollowerOfPerson
2019-07-04 14:36:29 +00:00
from pprint import pprint
2019-07-04 19:34:28 +00:00
from cache import getPersonFromCache
2019-07-04 20:25:19 +00:00
from cache import storePersonInCache
2019-07-06 15:17:21 +00:00
from acceptreject import receiveAcceptReject
2019-07-07 15:51:04 +00:00
from capabilities import getOcapFilename
2019-07-07 22:06:46 +00:00
from capabilities import CapablePost
2019-07-09 14:20:23 +00:00
from capabilities import capabilitiesReceiveUpdate
2019-07-10 12:40:31 +00:00
from like import updateLikesCollection
2019-07-12 09:10:09 +00:00
from like import undoLikesCollectionEntry
2019-11-17 14:02:59 +00:00
from bookmarks import updateBookmarksCollection
2019-11-17 14:01:49 +00:00
from bookmarks import undoBookmarksCollectionEntry
from blocking import isBlocked
2019-10-17 13:18:21 +00:00
from blocking import isBlockedDomain
2019-07-14 20:50:27 +00:00
from filters import isFiltered
from announce import updateAnnounceCollection
2019-10-21 11:02:58 +00:00
from announce import undoAnnounceCollectionEntry
from httpsig import messageContentDigest
2020-08-25 19:45:15 +00:00
from posts import validContentWarning
from posts import downloadAnnounce
from posts import isDM
from posts import isReply
2020-08-27 17:40:09 +00:00
from posts import isMuted
2019-10-22 20:30:43 +00:00
from posts import isImageMedia
2019-10-04 12:39:46 +00:00
from posts import sendSignedJson
from posts import sendToFollowersThread
from webinterface import individualPostAsHtml
from webinterface import getIconsDir
from webinterface import removeOldHashtags
2019-11-29 18:46:21 +00:00
from question import questionUpdateVotes
from media import replaceYouTube
2020-05-02 11:08:38 +00:00
from git import isGitPatch
from git import receiveGitPatch
2020-07-03 19:20:31 +00:00
from followingCalendar import receivingCalendarEvents
2020-07-10 14:15:01 +00:00
from content import dangerousMarkup
2020-08-20 16:51:48 +00:00
from happening import saveEventPost
2020-04-03 16:27:34 +00:00
2020-05-02 17:16:24 +00:00
2020-04-03 16:27:34 +00:00
def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
2019-12-12 17:34:31 +00:00
"""Extracts hashtags from an incoming post and updates the
relevant tags files.
"""
if not isPublicPost(postJsonObject):
return
if not postJsonObject.get('object'):
return
if not isinstance(postJsonObject['object'], dict):
return
if not postJsonObject['object'].get('tag'):
return
if not postJsonObject.get('id'):
return
if not isinstance(postJsonObject['object']['tag'], list):
return
2020-04-03 16:27:34 +00:00
tagsDir = baseDir+'/tags'
2019-12-12 17:34:31 +00:00
for tag in postJsonObject['object']['tag']:
if not tag.get('type'):
continue
2020-04-03 16:27:34 +00:00
if tag['type'] != 'Hashtag':
2019-12-12 17:34:31 +00:00
continue
if not tag.get('name'):
continue
2020-04-03 16:27:34 +00:00
tagName = tag['name'].replace('#', '').strip()
tagsFilename = tagsDir + '/' + tagName + '.txt'
2020-08-23 11:13:35 +00:00
postUrl = removeIdEnding(postJsonObject['id'])
2020-04-03 16:27:34 +00:00
postUrl = postUrl.replace('/', '#')
daysDiff = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
daysSinceEpoch = daysDiff.days
tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n'
2019-12-12 17:34:31 +00:00
if not os.path.isfile(tagsFilename):
2020-04-03 16:27:34 +00:00
tagsFile = open(tagsFilename, "w+")
2019-12-12 17:34:31 +00:00
if tagsFile:
2019-12-12 19:18:29 +00:00
tagsFile.write(tagline)
2019-12-12 17:34:31 +00:00
tagsFile.close()
else:
if postUrl not in open(tagsFilename).read():
2019-12-12 17:47:16 +00:00
try:
with open(tagsFilename, 'r+') as tagsFile:
2020-04-03 16:27:34 +00:00
content = tagsFile.read()
2019-12-12 17:47:16 +00:00
tagsFile.seek(0, 0)
2020-04-03 16:27:34 +00:00
tagsFile.write(tagline + content)
2019-12-12 17:49:16 +00:00
except Exception as e:
2020-04-03 16:27:34 +00:00
print('WARN: Failed to write entry to tags file ' +
tagsFilename + ' ' + str(e))
removeOldHashtags(baseDir, 3)
2020-04-03 16:27:34 +00:00
def inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
translate: {},
baseDir: str, httpPrefix: str,
session, cachedWebfingers: {}, personCache: {},
nickname: str, domain: str, port: int,
postJsonObject: {},
allowDeletion: bool, boxname: str) -> None:
"""Converts the json post into html and stores it in a cache
This enables the post to be quickly displayed later
"""
2020-04-03 16:27:34 +00:00
pageNumber = -999
avatarUrl = None
if boxname != 'tlevents' and boxname != 'outbox':
boxName = 'inbox'
2020-04-03 16:27:34 +00:00
individualPostAsHtml(recentPostsCache, maxRecentPosts,
getIconsDir(baseDir), translate, pageNumber,
baseDir, session, cachedWebfingers, personCache,
nickname, domain, port, postJsonObject,
avatarUrl, True, allowDeletion,
httpPrefix, __version__, boxName,
not isDM(postJsonObject),
True, True, False, True)
def validInbox(baseDir: str, nickname: str, domain: str) -> bool:
2019-07-18 11:35:48 +00:00
"""Checks whether files were correctly saved to the inbox
"""
if ':' in domain:
2020-04-03 16:27:34 +00:00
domain = domain.split(':')[0]
inboxDir = baseDir+'/accounts/' + nickname + '@' + domain + '/inbox'
if not os.path.isdir(inboxDir):
return True
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
2020-04-03 16:27:34 +00:00
filename = os.path.join(subdir, f)
if not os.path.isfile(filename):
2020-04-03 16:27:34 +00:00
print('filename: ' + filename)
return False
if 'postNickname' in open(filename).read():
2020-04-03 16:27:34 +00:00
print('queue file incorrectly saved to ' + filename)
2019-07-18 11:35:48 +00:00
return False
2020-03-22 21:16:02 +00:00
return True
2019-07-18 11:35:48 +00:00
2020-04-03 16:27:34 +00:00
def validInboxFilenames(baseDir: str, nickname: str, domain: str,
expectedDomain: str, expectedPort: int) -> bool:
2019-07-18 11:35:48 +00:00
"""Used by unit tests to check that the port number gets appended to
domain names within saved post filenames
"""
if ':' in domain:
2020-04-03 16:27:34 +00:00
domain = domain.split(':')[0]
inboxDir = baseDir + '/accounts/' + nickname + '@' + domain + '/inbox'
2019-07-18 11:35:48 +00:00
if not os.path.isdir(inboxDir):
return True
2020-04-03 16:27:34 +00:00
expectedStr = expectedDomain + ':' + str(expectedPort)
2019-07-18 11:35:48 +00:00
for subdir, dirs, files in os.walk(inboxDir):
for f in files:
2020-04-03 16:27:34 +00:00
filename = os.path.join(subdir, f)
2019-07-18 11:35:48 +00:00
if not os.path.isfile(filename):
2020-04-03 16:27:34 +00:00
print('filename: ' + filename)
2019-07-18 11:35:48 +00:00
return False
2020-04-03 16:27:34 +00:00
if expectedStr not in filename:
print('Expected: ' + expectedStr)
print('Invalid filename: ' + filename)
return False
2020-03-22 21:16:02 +00:00
return True
2020-04-03 16:27:34 +00:00
def getPersonPubKey(baseDir: str, session, personUrl: str,
personCache: {}, debug: bool,
projectVersion: str, httpPrefix: str,
domain: str, onionDomain: str) -> str:
2019-07-04 19:34:28 +00:00
if not personUrl:
return None
2020-04-03 16:27:34 +00:00
personUrl = personUrl.replace('#main-key', '')
2019-08-05 16:05:08 +00:00
if personUrl.endswith('/users/inbox'):
if debug:
print('DEBUG: Obtaining public key for shared inbox')
2020-04-03 16:27:34 +00:00
personUrl = personUrl.replace('/users/inbox', '/inbox')
personJson = \
getPersonFromCache(baseDir, personUrl, personCache, True)
2019-07-04 19:34:28 +00:00
if not personJson:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Obtaining public key for ' + personUrl)
personDomain = domain
2020-03-02 13:35:24 +00:00
if onionDomain:
if '.onion/' in personUrl:
2020-04-03 16:27:34 +00:00
personDomain = onionDomain
profileStr = 'https://www.w3.org/ns/activitystreams'
asHeader = {
'Accept': 'application/activity+json; profile="' + profileStr + '"'
}
2020-04-03 16:27:34 +00:00
personJson = \
getJson(session, personUrl, asHeader, None, projectVersion,
httpPrefix, personDomain)
2019-07-04 19:34:28 +00:00
if not personJson:
return None
2020-04-03 16:27:34 +00:00
pubKey = None
2019-07-04 19:34:28 +00:00
if personJson.get('publicKey'):
if personJson['publicKey'].get('publicKeyPem'):
2020-04-03 16:27:34 +00:00
pubKey = personJson['publicKey']['publicKeyPem']
2019-07-04 19:34:28 +00:00
else:
if personJson.get('publicKeyPem'):
2020-04-03 16:27:34 +00:00
pubKey = personJson['publicKeyPem']
2019-07-04 19:34:28 +00:00
if not pubKey:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Public key not found for ' + personUrl)
2019-07-04 19:34:28 +00:00
storePersonInCache(baseDir, personUrl, personJson, personCache, True)
2019-07-04 19:34:28 +00:00
return pubKey
2019-06-28 21:59:54 +00:00
2020-04-03 16:27:34 +00:00
2019-07-02 15:07:27 +00:00
def inboxMessageHasParams(messageJson: {}) -> bool:
"""Checks whether an incoming message contains expected parameters
"""
2020-08-23 14:45:58 +00:00
expectedParams = ['actor', 'type', 'object']
2019-07-02 15:07:27 +00:00
for param in expectedParams:
if not messageJson.get(param):
2020-08-23 14:45:58 +00:00
# print('inboxMessageHasParams: ' +
# param + ' ' + str(messageJson))
2019-07-02 15:07:27 +00:00
return False
2019-07-06 13:49:25 +00:00
if not messageJson.get('to'):
2020-04-03 16:27:34 +00:00
allowedWithoutToParam = ['Like', 'Follow', 'Request',
'Accept', 'Capability', 'Undo']
2019-07-06 13:49:25 +00:00
if messageJson['type'] not in allowedWithoutToParam:
return False
2019-07-02 15:07:27 +00:00
return True
2020-04-03 16:27:34 +00:00
def inboxPermittedMessage(domain: str, messageJson: {},
federationList: []) -> bool:
2019-06-28 21:59:54 +00:00
""" check that we are receiving from a permitted domain
"""
2019-11-16 12:30:59 +00:00
if not messageJson.get('actor'):
2019-06-28 21:59:54 +00:00
return False
2020-08-23 14:45:58 +00:00
2020-04-03 16:27:34 +00:00
actor = messageJson['actor']
2019-06-28 21:59:54 +00:00
# always allow the local domain
2019-07-01 11:48:54 +00:00
if domain in actor:
2019-06-28 21:59:54 +00:00
return True
2020-04-03 16:27:34 +00:00
if not urlPermitted(actor, federationList, "inbox:write"):
2019-06-28 21:59:54 +00:00
return False
2020-04-03 16:27:34 +00:00
alwaysAllowedTypes = ('Follow', 'Like', 'Delete', 'Announce')
2019-11-16 12:30:59 +00:00
if messageJson['type'] not in alwaysAllowedTypes:
2019-11-16 12:32:28 +00:00
if not messageJson.get('object'):
return True
if not isinstance(messageJson['object'], dict):
return False
if messageJson['object'].get('inReplyTo'):
2020-04-03 16:27:34 +00:00
inReplyTo = messageJson['object']['inReplyTo']
2020-08-28 14:45:07 +00:00
if not isinstance(inReplyTo, str):
return False
2020-04-03 16:27:34 +00:00
if not urlPermitted(inReplyTo, federationList, "inbox:write"):
2019-07-15 09:20:16 +00:00
return False
2019-06-28 21:59:54 +00:00
return True
2019-06-29 10:08:59 +00:00
2020-04-03 16:27:34 +00:00
2019-10-11 16:54:55 +00:00
def validPublishedDate(published: str) -> bool:
2020-04-03 16:27:34 +00:00
currTime = datetime.datetime.utcnow()
pubDate = datetime.datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
daysSincePublished = (currTime - pubDate).days
if daysSincePublished > 30:
2019-06-29 10:08:59 +00:00
return False
return True
2019-07-04 10:02:56 +00:00
2020-04-03 16:27:34 +00:00
def savePostToInboxQueue(baseDir: str, httpPrefix: str,
nickname: str, domain: str,
postJsonObject: {},
messageBytes: str,
httpHeaders: {},
postPath: str, debug: bool) -> str:
2019-07-04 10:02:56 +00:00
"""Saves the give json to the inbox queue for the person
keyId specifies the actor sending the post
"""
2020-04-03 16:27:34 +00:00
if len(messageBytes) > 10240:
print('WARN: inbox message too long ' +
str(len(messageBytes)) + ' bytes')
return None
2020-04-03 16:27:34 +00:00
originalDomain = domain
2019-07-04 10:02:56 +00:00
if ':' in domain:
2020-04-03 16:27:34 +00:00
domain = domain.split(':')[0]
# block at the ealiest stage possible, which means the data
# isn't written to file
2020-04-03 16:27:34 +00:00
postNickname = None
postDomain = None
actor = None
if postJsonObject.get('actor'):
2020-04-03 16:27:34 +00:00
actor = postJsonObject['actor']
postNickname = getNicknameFromActor(postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
if not postNickname:
2020-04-03 16:27:34 +00:00
print('No post Nickname in actor ' + postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
return None
2020-04-03 16:27:34 +00:00
postDomain, postPort = getDomainFromActor(postJsonObject['actor'])
2019-09-01 19:20:28 +00:00
if not postDomain:
2019-10-29 20:23:49 +00:00
if debug:
pprint(postJsonObject)
2019-09-01 19:20:28 +00:00
print('No post Domain in actor')
return None
2020-04-03 16:27:34 +00:00
if isBlocked(baseDir, nickname, domain, postNickname, postDomain):
2019-08-18 09:39:12 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: post from ' + postNickname + ' blocked')
return None
2019-07-15 10:22:19 +00:00
if postPort:
2020-04-03 16:27:34 +00:00
if postPort != 80 and postPort != 443:
if ':' not in postDomain:
2020-04-03 16:27:34 +00:00
postDomain = postDomain + ':' + str(postPort)
2019-07-14 20:50:27 +00:00
2019-08-05 09:28:12 +00:00
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('inReplyTo'):
if isinstance(postJsonObject['object']['inReplyTo'], str):
2020-04-03 16:27:34 +00:00
inReplyTo = \
postJsonObject['object']['inReplyTo']
replyDomain, replyPort = \
getDomainFromActor(inReplyTo)
if isBlockedDomain(baseDir, replyDomain):
print('WARN: post contains reply from ' +
str(actor) +
' to a blocked domain: ' + replyDomain)
return None
2019-10-17 13:18:21 +00:00
else:
2020-04-03 16:27:34 +00:00
replyNickname = \
getNicknameFromActor(inReplyTo)
2019-10-17 13:18:21 +00:00
if replyNickname and replyDomain:
2020-04-03 16:27:34 +00:00
if isBlocked(baseDir, nickname, domain,
replyNickname, replyDomain):
print('WARN: post contains reply from ' +
str(actor) +
' to a blocked account: ' +
replyNickname + '@' + replyDomain)
2019-10-17 13:18:21 +00:00
return None
2019-08-05 09:28:12 +00:00
if postJsonObject['object'].get('content'):
if isinstance(postJsonObject['object']['content'], str):
2020-04-03 16:27:34 +00:00
if isFiltered(baseDir, nickname, domain,
postJsonObject['object']['content']):
2019-09-01 19:54:02 +00:00
print('WARN: post was filtered out due to content')
2019-08-05 09:28:12 +00:00
return None
2020-04-03 16:27:34 +00:00
originalPostId = None
2019-07-14 16:57:06 +00:00
if postJsonObject.get('id'):
2020-08-23 11:13:35 +00:00
originalPostId = removeIdEnding(postJsonObject['id'])
2019-08-16 15:04:40 +00:00
2020-04-03 16:27:34 +00:00
currTime = datetime.datetime.utcnow()
2019-08-16 15:04:40 +00:00
2020-04-03 16:27:34 +00:00
postId = None
2019-08-16 15:04:40 +00:00
if postJsonObject.get('id'):
2020-08-23 11:13:35 +00:00
postId = removeIdEnding(postJsonObject['id'])
2020-04-03 16:27:34 +00:00
published = currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
2019-08-16 15:04:40 +00:00
if not postId:
2020-04-03 16:27:34 +00:00
statusNumber, published = getStatusNumber()
2019-08-16 15:04:40 +00:00
if actor:
2020-04-03 16:27:34 +00:00
postId = actor + '/statuses/' + statusNumber
2019-08-16 15:04:40 +00:00
else:
2020-04-03 16:27:34 +00:00
postId = httpPrefix + '://' + originalDomain + \
'/users/' + nickname + '/statuses/' + statusNumber
2020-03-22 21:16:02 +00:00
2019-08-16 13:47:01 +00:00
# NOTE: don't change postJsonObject['id'] before signature check
2020-03-22 21:16:02 +00:00
2020-04-03 16:27:34 +00:00
inboxQueueDir = createInboxQueueDir(nickname, domain, baseDir)
handle = nickname + '@' + domain
destination = baseDir + '/accounts/' + \
handle + '/inbox/' + postId.replace('/', '#') + '.json'
filename = inboxQueueDir + '/' + postId.replace('/', '#') + '.json'
sharedInboxItem = False
if nickname == 'inbox':
nickname = originalDomain
sharedInboxItem = True
digestStartTime = time.time()
digest = messageContentDigest(messageBytes)
timeDiffStr = str(int((time.time() - digestStartTime) * 1000))
2019-11-16 10:12:40 +00:00
if debug:
2020-04-03 16:27:34 +00:00
while len(timeDiffStr) < 6:
timeDiffStr = '0' + timeDiffStr
print('DIGEST|' + timeDiffStr + '|' + filename)
2019-11-16 10:07:32 +00:00
2020-04-03 16:27:34 +00:00
newQueueItem = {
2019-08-16 09:35:06 +00:00
'originalId': originalPostId,
2019-07-15 09:20:16 +00:00
'id': postId,
2019-08-16 09:35:06 +00:00
'actor': actor,
2019-07-07 15:51:04 +00:00
'nickname': nickname,
'domain': domain,
2019-07-15 10:22:19 +00:00
'postNickname': postNickname,
'postDomain': postDomain,
'sharedInbox': sharedInboxItem,
2019-07-04 10:09:27 +00:00
'published': published,
2019-08-15 21:34:25 +00:00
'httpHeaders': httpHeaders,
2019-07-05 22:13:20 +00:00
'path': postPath,
2019-07-14 16:57:06 +00:00
'post': postJsonObject,
2019-11-16 10:07:32 +00:00
'digest': digest,
'filename': filename,
2019-08-05 09:50:45 +00:00
'destination': destination
2019-07-04 10:02:56 +00:00
}
2019-07-06 13:49:25 +00:00
if debug:
print('Inbox queue item created')
2020-04-03 16:27:34 +00:00
saveJson(newQueueItem, filename)
2019-07-04 10:02:56 +00:00
return filename
2019-07-04 12:23:53 +00:00
2020-04-03 16:27:34 +00:00
def inboxCheckCapabilities(baseDir: str, nickname: str, domain: str,
actor: str, queueFilename: str, queue: [],
queueJson: {}, capabilityId: str,
debug: bool) -> bool:
if nickname == 'inbox':
2019-07-08 18:55:39 +00:00
return True
2020-04-03 16:27:34 +00:00
ocapFilename = \
getOcapFilename(baseDir,
queueJson['nickname'], queueJson['domain'],
actor, 'accept')
2019-08-18 20:47:12 +00:00
if not ocapFilename:
return False
2019-07-08 18:55:39 +00:00
if not os.path.isfile(ocapFilename):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: capabilities for ' +
actor + ' do not exist')
2020-02-21 11:32:43 +00:00
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
2020-02-21 11:32:43 +00:00
queue.pop(0)
return False
2019-07-08 18:55:39 +00:00
2020-04-03 16:27:34 +00:00
oc = loadJson(ocapFilename)
2020-03-22 21:16:02 +00:00
if not oc:
return False
2019-07-08 18:55:39 +00:00
if not oc.get('id'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: capabilities for ' + actor + ' do not contain an id')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
queue.pop(0)
2019-07-08 18:55:39 +00:00
return False
2020-04-03 16:27:34 +00:00
if oc['id'] != capabilityId:
2019-07-08 18:55:39 +00:00
if debug:
print('DEBUG: capability id mismatch')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
queue.pop(0)
2019-07-08 18:55:39 +00:00
return False
if not oc.get('capability'):
if debug:
print('DEBUG: missing capability list')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
queue.pop(0)
2019-07-08 18:55:39 +00:00
return False
2020-04-03 16:27:34 +00:00
if not CapablePost(queueJson['post'], oc['capability'], debug):
2019-07-08 18:55:39 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: insufficient capabilities to write to inbox from ' +
actor)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
queue.pop(0)
2019-07-08 18:55:39 +00:00
return False
if debug:
print('DEBUG: object capabilities check success')
return True
2020-04-03 16:27:34 +00:00
def inboxPostRecipientsAdd(baseDir: str, httpPrefix: str, toList: [],
recipientsDict: {},
domainMatch: str, domain: str,
actor: str, debug: bool) -> bool:
2019-07-08 22:12:24 +00:00
"""Given a list of post recipients (toList) from 'to' or 'cc' parameters
populate a recipientsDict with the handle and capabilities id for each
"""
2020-04-03 16:27:34 +00:00
followerRecipients = False
2019-07-08 22:12:24 +00:00
for recipient in toList:
2019-09-03 19:53:22 +00:00
if not recipient:
continue
2019-07-08 22:12:24 +00:00
# is this a to a local account?
if domainMatch in recipient:
# get the handle for the local account
2020-04-03 16:27:34 +00:00
nickname = recipient.split(domainMatch)[1]
handle = nickname+'@'+domain
if os.path.isdir(baseDir + '/accounts/' + handle):
2019-07-08 22:12:24 +00:00
# are capabilities granted for this account to the
# sender (actor) of the post?
2020-04-03 16:27:34 +00:00
ocapFilename = \
baseDir + '/accounts/' + handle + \
'/ocap/accept/' + actor.replace('/', '#') + '.json'
2019-07-08 22:12:24 +00:00
if os.path.isfile(ocapFilename):
# read the granted capabilities and obtain the id
2020-04-03 16:27:34 +00:00
ocapJson = loadJson(ocapFilename)
2019-10-22 11:55:06 +00:00
if ocapJson:
2019-07-08 22:12:24 +00:00
if ocapJson.get('id'):
# append with the capabilities id
2020-04-03 16:27:34 +00:00
recipientsDict[handle] = ocapJson['id']
2019-07-08 22:12:24 +00:00
else:
2020-04-03 16:27:34 +00:00
recipientsDict[handle] = None
2019-07-08 22:12:24 +00:00
else:
2019-07-11 12:29:31 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + ocapFilename + ' not found')
recipientsDict[handle] = None
2019-07-11 12:29:31 +00:00
else:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + baseDir + '/accounts/' +
handle + ' does not exist')
2019-07-11 12:29:31 +00:00
else:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + recipient + ' is not local to ' +
domainMatch)
2019-07-11 12:29:31 +00:00
print(str(toList))
2019-07-08 22:12:24 +00:00
if recipient.endswith('followers'):
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: followers detected as post recipients')
2020-04-03 16:27:34 +00:00
followerRecipients = True
return followerRecipients, recipientsDict
2019-07-08 22:12:24 +00:00
2020-04-03 16:27:34 +00:00
def inboxPostRecipients(baseDir: str, postJsonObject: {},
httpPrefix: str, domain: str, port: int,
debug: bool) -> ([], []):
"""Returns dictionaries containing the recipients of the given post
The shared dictionary contains followers
"""
2020-04-03 16:27:34 +00:00
recipientsDict = {}
recipientsDictFollowers = {}
2019-07-08 22:12:24 +00:00
if not postJsonObject.get('actor'):
2019-07-11 12:29:31 +00:00
if debug:
pprint(postJsonObject)
print('WARNING: inbox post has no actor')
2020-04-03 16:27:34 +00:00
return recipientsDict, recipientsDictFollowers
2019-07-08 22:12:24 +00:00
if ':' in domain:
2020-04-03 16:27:34 +00:00
domain = domain.split(':')[0]
domainBase = domain
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
if ':' not in domain:
2020-04-03 16:27:34 +00:00
domain = domain + ':' + str(port)
domainMatch = '/' + domain + '/users/'
2019-07-08 22:12:24 +00:00
2020-04-03 16:27:34 +00:00
actor = postJsonObject['actor']
2019-07-08 22:12:24 +00:00
# first get any specific people which the post is addressed to
2020-03-22 21:16:02 +00:00
2020-04-03 16:27:34 +00:00
followerRecipients = False
2019-07-08 22:12:24 +00:00
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('to'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['object']['to'], list):
2020-04-03 16:27:34 +00:00
recipientsList = postJsonObject['object']['to']
2019-08-16 17:51:00 +00:00
else:
2020-04-03 16:27:34 +00:00
recipientsList = [postJsonObject['object']['to']]
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: resolving "to"')
2020-04-03 16:27:34 +00:00
includesFollowers, recipientsDict = \
inboxPostRecipientsAdd(baseDir, httpPrefix,
recipientsList,
recipientsDict,
domainMatch, domainBase,
actor, debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
2020-04-03 16:27:34 +00:00
followerRecipients = True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no "to"')
2019-07-08 22:12:24 +00:00
if postJsonObject['object'].get('cc'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['object']['cc'], list):
2020-04-03 16:27:34 +00:00
recipientsList = postJsonObject['object']['cc']
2019-08-16 17:51:00 +00:00
else:
2020-04-03 16:27:34 +00:00
recipientsList = [postJsonObject['object']['cc']]
includesFollowers, recipientsDict = \
inboxPostRecipientsAdd(baseDir, httpPrefix,
recipientsList,
recipientsDict,
domainMatch, domainBase,
actor, debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
2020-04-03 16:27:34 +00:00
followerRecipients = True
2019-07-11 12:29:31 +00:00
else:
if debug:
print('DEBUG: inbox post has no cc')
else:
if debug:
if isinstance(postJsonObject['object'], str):
if '/statuses/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to a post')
else:
if '/users/' in postJsonObject['object']:
print('DEBUG: inbox item is a link to an actor')
2019-07-08 22:12:24 +00:00
if postJsonObject.get('to'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['to'], list):
2020-04-03 16:27:34 +00:00
recipientsList = postJsonObject['to']
2019-08-16 17:51:00 +00:00
else:
2020-04-03 16:27:34 +00:00
recipientsList = [postJsonObject['to']]
includesFollowers, recipientsDict = \
inboxPostRecipientsAdd(baseDir, httpPrefix,
recipientsList,
recipientsDict,
domainMatch, domainBase,
actor, debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
2020-04-03 16:27:34 +00:00
followerRecipients = True
2019-07-08 22:12:24 +00:00
if postJsonObject.get('cc'):
2019-08-16 17:51:00 +00:00
if isinstance(postJsonObject['cc'], list):
2020-04-03 16:27:34 +00:00
recipientsList = postJsonObject['cc']
2019-08-16 17:51:00 +00:00
else:
2020-04-03 16:27:34 +00:00
recipientsList = [postJsonObject['cc']]
includesFollowers, recipientsDict = \
inboxPostRecipientsAdd(baseDir, httpPrefix,
recipientsList,
recipientsDict,
domainMatch, domainBase,
actor, debug)
2019-07-08 22:12:24 +00:00
if includesFollowers:
2020-04-03 16:27:34 +00:00
followerRecipients = True
2019-07-08 22:12:24 +00:00
if not followerRecipients:
2019-07-11 12:29:31 +00:00
if debug:
print('DEBUG: no followers were resolved')
2020-04-03 16:27:34 +00:00
return recipientsDict, recipientsDictFollowers
2019-07-08 22:12:24 +00:00
# now resolve the followers
2020-04-03 16:27:34 +00:00
recipientsDictFollowers = \
getFollowersOfActor(baseDir, actor, debug)
return recipientsDict, recipientsDictFollowers
2019-07-08 22:12:24 +00:00
2020-04-03 16:27:34 +00:00
def receiveUndoFollow(session, baseDir: str, httpPrefix: str,
port: int, messageJson: {},
federationList: [],
debug: bool) -> bool:
2019-07-17 10:34:00 +00:00
if not messageJson['object'].get('actor'):
if debug:
print('DEBUG: follow request has no actor within object')
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['object']['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['object']['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['object']['actor'] and \
'/profile/' not in messageJson['object']['actor']:
2019-07-17 10:34:00 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" or "profile" missing ' +
'from actor within object')
2019-07-17 10:34:00 +00:00
return False
if messageJson['object']['actor'] != messageJson['actor']:
if debug:
print('DEBUG: actors do not match')
return False
2020-04-03 16:27:34 +00:00
nicknameFollower = \
getNicknameFromActor(messageJson['object']['actor'])
2019-09-02 09:43:43 +00:00
if not nicknameFollower:
2020-04-03 16:27:34 +00:00
print('WARN: unable to find nickname in ' +
messageJson['object']['actor'])
2019-09-02 09:43:43 +00:00
return False
2020-04-03 16:27:34 +00:00
domainFollower, portFollower = \
getDomainFromActor(messageJson['object']['actor'])
domainFollowerFull = domainFollower
2019-07-17 10:34:00 +00:00
if portFollower:
2020-04-03 16:27:34 +00:00
if portFollower != 80 and portFollower != 443:
if ':' not in domainFollower:
2020-04-03 16:27:34 +00:00
domainFollowerFull = domainFollower + ':' + str(portFollower)
2020-03-22 21:16:02 +00:00
2020-04-03 16:27:34 +00:00
nicknameFollowing = \
getNicknameFromActor(messageJson['object']['object'])
2019-09-02 09:43:43 +00:00
if not nicknameFollowing:
2020-04-03 16:27:34 +00:00
print('WARN: unable to find nickname in ' +
messageJson['object']['object'])
2019-09-02 09:43:43 +00:00
return False
2020-04-03 16:27:34 +00:00
domainFollowing, portFollowing = \
getDomainFromActor(messageJson['object']['object'])
domainFollowingFull = domainFollowing
2019-07-17 10:34:00 +00:00
if portFollowing:
2020-04-03 16:27:34 +00:00
if portFollowing != 80 and portFollowing != 443:
if ':' not in domainFollowing:
2020-04-03 16:27:34 +00:00
domainFollowingFull = \
domainFollowing + ':' + str(portFollowing)
2019-07-17 10:34:00 +00:00
2020-04-03 16:27:34 +00:00
if unfollowerOfPerson(baseDir,
nicknameFollowing, domainFollowingFull,
nicknameFollower, domainFollowerFull,
2019-07-17 11:54:13 +00:00
debug):
2020-08-20 12:11:07 +00:00
print(nicknameFollowing + '@' + domainFollowingFull + ': '
'Follower ' + nicknameFollower + '@' + domainFollowerFull +
' was removed')
2019-07-17 11:54:13 +00:00
return True
2020-03-22 21:16:02 +00:00
2019-07-17 11:54:13 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Follower ' +
nicknameFollower + '@' + domainFollowerFull +
' was not removed')
2019-07-17 11:54:13 +00:00
return False
2019-07-17 10:34:00 +00:00
2020-04-03 16:27:34 +00:00
def receiveUndo(session, baseDir: str, httpPrefix: str,
port: int, sendThreads: [], postLog: [],
cachedWebfingers: {}, personCache: {},
messageJson: {}, federationList: [],
debug: bool,
acceptedCaps=["inbox:write", "objects:read"]) -> bool:
2019-07-17 10:34:00 +00:00
"""Receives an undo request within the POST section of HTTPServer
"""
if not messageJson['type'].startswith('Undo'):
return False
2019-07-17 11:24:11 +00:00
if debug:
print('DEBUG: Undo activity received')
2019-07-17 10:34:00 +00:00
if not messageJson.get('actor'):
if debug:
print('DEBUG: follow request has no actor')
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-17 10:34:00 +00:00
if debug:
2020-03-22 21:16:02 +00:00
print('DEBUG: "users" or "profile" missing from actor')
2019-07-17 10:34:00 +00:00
return False
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-07-17 10:34:00 +00:00
return False
if not isinstance(messageJson['object'], dict):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
2019-07-17 10:34:00 +00:00
return False
if not messageJson['object'].get('type'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object type')
2019-07-17 10:34:00 +00:00
return False
if not messageJson['object'].get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] +
' has no object within object')
2019-07-17 10:34:00 +00:00
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] +
' object within object is not a string')
2019-07-17 10:34:00 +00:00
return False
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] == 'Follow':
return receiveUndoFollow(session, baseDir, httpPrefix,
port, messageJson,
federationList, debug)
2019-07-17 10:34:00 +00:00
return False
2020-04-03 16:27:34 +00:00
2020-08-20 16:51:48 +00:00
def receiveEventPost(recentPostsCache: {}, session, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
nickname: str, debug: bool) -> bool:
"""Receive a mobilizon-type event activity
2020-08-20 17:08:25 +00:00
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
2020-08-20 16:51:48 +00:00
"""
if not isEventPost(messageJson):
return
print('Receiving event: ' + str(messageJson['object']))
handle = nickname + '@' + domain
if port:
if port != 80 and port != 443:
handle += ':' + str(port)
2020-08-23 11:13:35 +00:00
postId = removeIdEnding(messageJson['id']).replace('/', '#')
2020-08-20 16:51:48 +00:00
saveEventPost(baseDir, handle, postId, messageJson['object'])
2020-04-03 16:27:34 +00:00
def personReceiveUpdate(baseDir: str,
domain: str, port: int,
updateNickname: str, updateDomain: str,
updatePort: int,
personJson: {}, personCache: {}, debug: bool) -> bool:
"""Changes an actor. eg: avatar or display name change
2019-08-20 19:41:58 +00:00
"""
if debug:
print('DEBUG: receiving actor update for '+personJson['url'])
2020-04-03 16:27:34 +00:00
domainFull = domain
2019-08-20 19:41:58 +00:00
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
domainFull = domain + ':' + str(port)
updateDomainFull = updateDomain
if updatePort:
2020-04-03 16:27:34 +00:00
if updatePort != 80 and updatePort != 443:
updateDomainFull = updateDomain + ':' + str(updatePort)
actor = updateDomainFull + '/users/' + updateNickname
if actor not in personJson['id']:
2020-04-03 16:27:34 +00:00
actor = updateDomainFull + '/profile/' + updateNickname
if actor not in personJson['id']:
2020-04-03 16:27:34 +00:00
actor = updateDomainFull + '/channel/' + updateNickname
2019-10-17 22:26:47 +00:00
if actor not in personJson['id']:
2020-08-13 16:19:35 +00:00
actor = updateDomainFull + '/accounts/' + updateNickname
if actor not in personJson['id']:
if debug:
print('actor: ' + actor)
print('id: ' + personJson['id'])
print('DEBUG: Actor does not match id')
return False
2020-04-03 16:27:34 +00:00
if updateDomainFull == domainFull:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: You can only receive actor updates ' +
'for domains other than your own')
2019-08-20 19:41:58 +00:00
return False
if not personJson.get('publicKey'):
if debug:
2020-03-22 21:16:02 +00:00
print('DEBUG: actor update does not contain a public key')
2019-08-20 19:41:58 +00:00
return False
if not personJson['publicKey'].get('publicKeyPem'):
if debug:
2020-03-22 21:16:02 +00:00
print('DEBUG: actor update does not contain a public key Pem')
2019-08-20 19:41:58 +00:00
return False
2020-04-03 16:27:34 +00:00
actorFilename = baseDir + '/cache/actors/' + \
personJson['id'].replace('/', '#') + '.json'
2019-08-20 19:41:58 +00:00
# check that the public keys match.
# If they don't then this may be a nefarious attempt to hack an account
2020-04-03 16:27:34 +00:00
idx = personJson['id']
if personCache.get(idx):
if personCache[idx]['actor']['publicKey']['publicKeyPem'] != \
personJson['publicKey']['publicKeyPem']:
2019-08-20 19:41:58 +00:00
if debug:
print('WARN: Public key does not match when updating actor')
return False
else:
if os.path.isfile(actorFilename):
2020-04-03 16:27:34 +00:00
existingPersonJson = loadJson(actorFilename)
2019-10-22 11:55:06 +00:00
if existingPersonJson:
2020-04-03 16:27:34 +00:00
if existingPersonJson['publicKey']['publicKeyPem'] != \
personJson['publicKey']['publicKeyPem']:
2019-08-20 19:41:58 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('WARN: Public key does not match ' +
'cached actor when updating')
2019-08-20 19:41:58 +00:00
return False
# save to cache in memory
storePersonInCache(baseDir, personJson['id'], personJson,
personCache, True)
2019-08-20 19:41:58 +00:00
# save to cache on file
2020-04-03 16:27:34 +00:00
if saveJson(personJson, actorFilename):
print('actor updated for ' + personJson['id'])
# remove avatar if it exists so that it will be refreshed later
# when a timeline is constructed
2020-04-03 16:27:34 +00:00
actorStr = personJson['id'].replace('/', '-')
removeAvatarFromCache(baseDir, actorStr)
2019-08-20 19:41:58 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveUpdateToQuestion(recentPostsCache: {}, messageJson: {},
baseDir: str, nickname: str, domain: str) -> None:
2019-11-26 10:43:37 +00:00
"""Updating a question as new votes arrive
"""
# message url of the question
if not messageJson.get('id'):
return
if not messageJson.get('actor'):
return
2020-08-23 11:13:35 +00:00
messageId = removeIdEnding(messageJson['id'])
2019-11-26 10:43:37 +00:00
if '#' in messageId:
2020-04-03 16:27:34 +00:00
messageId = messageId.split('#', 1)[0]
2019-11-26 10:43:37 +00:00
# find the question post
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, nickname, domain, messageId)
2019-11-26 10:43:37 +00:00
if not postFilename:
return
# load the json for the question
2020-04-03 16:27:34 +00:00
postJsonObject = loadJson(postFilename, 1)
2019-11-26 10:43:37 +00:00
if not postJsonObject:
return
if not postJsonObject.get('actor'):
return
# does the actor match?
2020-04-03 16:27:34 +00:00
if postJsonObject['actor'] != messageJson['actor']:
2019-11-26 10:43:37 +00:00
return
2020-04-03 16:27:34 +00:00
saveJson(messageJson, postFilename)
2019-11-26 10:43:37 +00:00
# ensure that the cached post is removed if it exists, so
# that it then will be recreated
2020-04-03 16:27:34 +00:00
cachedPostFilename = \
getCachedPostFilename(baseDir, nickname, domain, messageJson)
2019-11-26 10:43:37 +00:00
if cachedPostFilename:
if os.path.isfile(cachedPostFilename):
os.remove(cachedPostFilename)
# remove from memory cache
2020-04-03 16:27:34 +00:00
removePostFromCache(messageJson, recentPostsCache)
2020-03-22 21:16:02 +00:00
2020-04-03 16:27:34 +00:00
def receiveUpdate(recentPostsCache: {}, session, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
nickname: str, debug: bool) -> bool:
2019-07-09 14:20:23 +00:00
"""Receives an Update activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Update':
2019-07-09 14:20:23 +00:00
return False
if not messageJson.get('actor'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no actor')
2019-07-09 14:20:23 +00:00
return False
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-07-09 14:20:23 +00:00
return False
if not isinstance(messageJson['object'], dict):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
2019-07-09 14:20:23 +00:00
return False
if not messageJson['object'].get('type'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object has no type')
2019-07-09 14:20:23 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-09 14:20:23 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" or "profile" missing from actor in ' +
messageJson['type'])
2019-07-09 14:20:23 +00:00
return False
2019-08-22 17:25:12 +00:00
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] == 'Question':
receiveUpdateToQuestion(recentPostsCache, messageJson,
baseDir, nickname, domain)
if debug:
print('DEBUG: Question update was received')
return True
2020-04-03 16:27:34 +00:00
if messageJson['type'] == 'Person':
2020-01-19 21:05:02 +00:00
if messageJson.get('url') and messageJson.get('id'):
2020-04-03 16:27:34 +00:00
print('Request to update unwrapped actor: ' + messageJson['id'])
updateNickname = getNicknameFromActor(messageJson['id'])
2020-01-19 21:05:02 +00:00
if updateNickname:
2020-04-03 16:27:34 +00:00
updateDomain, updatePort = \
getDomainFromActor(messageJson['id'])
if personReceiveUpdate(baseDir, domain, port,
updateNickname, updateDomain,
updatePort, messageJson,
personCache, debug):
2020-01-19 21:05:02 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'Unwrapped profile update was received for ' +
messageJson['url'])
2020-01-19 21:05:02 +00:00
return True
2020-03-22 21:16:02 +00:00
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] == 'Person' or \
messageJson['object']['type'] == 'Application' or \
messageJson['object']['type'] == 'Group' or \
messageJson['object']['type'] == 'Service':
if messageJson['object'].get('url') and \
messageJson['object'].get('id'):
print('Request to update actor: ' + messageJson['actor'])
updateNickname = getNicknameFromActor(messageJson['actor'])
2019-09-02 09:43:43 +00:00
if updateNickname:
2020-04-03 16:27:34 +00:00
updateDomain, updatePort = \
getDomainFromActor(messageJson['actor'])
if personReceiveUpdate(baseDir,
domain, port,
updateNickname, updateDomain,
updatePort,
messageJson['object'],
personCache, debug):
2019-09-02 09:43:43 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Profile update was received for ' +
messageJson['object']['url'])
2019-09-02 09:43:43 +00:00
return True
2019-08-22 17:25:12 +00:00
2020-04-03 16:27:34 +00:00
if messageJson['object'].get('capability') and \
messageJson['object'].get('scope'):
nickname = getNicknameFromActor(messageJson['object']['scope'])
2019-09-02 09:43:43 +00:00
if nickname:
2020-04-03 16:27:34 +00:00
domain, tempPort = \
getDomainFromActor(messageJson['object']['scope'])
if messageJson['object']['type'] == 'Capability':
capability = messageJson['object']['capability']
if capabilitiesReceiveUpdate(baseDir, nickname, domain, port,
messageJson['actor'],
messageJson['object']['id'],
capability,
2019-09-02 09:43:43 +00:00
debug):
if debug:
print('DEBUG: An update was received')
return True
2019-07-09 14:20:23 +00:00
return False
2020-04-03 16:27:34 +00:00
def receiveLike(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
onionDomain: str,
2020-04-03 16:27:34 +00:00
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool) -> bool:
2019-07-10 12:40:31 +00:00
"""Receives a Like activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Like':
2019-07-10 12:40:31 +00:00
return False
if not messageJson.get('actor'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no actor')
2019-07-10 12:40:31 +00:00
return False
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-07-10 12:40:31 +00:00
return False
if not isinstance(messageJson['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a string')
2019-07-10 12:40:31 +00:00
return False
if not messageJson.get('to'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
2019-07-10 12:40:31 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-10 12:40:31 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" or "profile" missing from actor in ' +
messageJson['type'])
2019-07-10 12:40:31 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "statuses" missing from object in ' +
messageJson['type'])
2019-07-10 12:40:31 +00:00
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of like - ' + handle)
# if this post in the outbox of the person?
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, handle.split('@')[0],
handle.split('@')[1],
messageJson['object'])
2019-07-10 12:40:31 +00:00
if not postFilename:
if debug:
print('DEBUG: post not found in inbox or outbox')
print(messageJson['object'])
return True
if debug:
2019-07-11 12:59:00 +00:00
print('DEBUG: liked post found in inbox')
2019-10-19 17:50:05 +00:00
2020-07-13 13:36:45 +00:00
if not alreadyLiked(baseDir,
handle.split('@')[0],
handle.split('@')[1],
messageJson['object'],
messageJson['actor']):
updateLikesCollection(recentPostsCache, baseDir, postFilename,
messageJson['object'],
messageJson['actor'], domain, debug)
likeNotify(baseDir, domain, onionDomain, handle,
messageJson['actor'], messageJson['object'])
2019-07-10 12:40:31 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveUndoLike(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool) -> bool:
2019-07-12 09:10:09 +00:00
"""Receives an undo like activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Undo':
2019-07-12 09:10:09 +00:00
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('type'):
return False
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] != 'Like':
2019-07-12 09:10:09 +00:00
return False
if not messageJson['object'].get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' like has no object')
2019-07-12 09:10:09 +00:00
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] +
' like object is not a string')
2019-07-12 09:10:09 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-12 09:10:09 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" or "profile" missing from actor in ' +
messageJson['type'] + ' like')
2019-07-12 09:10:09 +00:00
return False
if '/statuses/' not in messageJson['object']['object']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "statuses" missing from like object in ' +
messageJson['type'])
2019-07-12 09:10:09 +00:00
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of undo like - ' + handle)
2019-07-12 09:10:09 +00:00
# if this post in the outbox of the person?
2020-04-03 16:27:34 +00:00
postFilename = \
locatePost(baseDir, handle.split('@')[0], handle.split('@')[1],
messageJson['object']['object'])
2019-07-12 09:10:09 +00:00
if not postFilename:
if debug:
2019-07-12 09:41:57 +00:00
print('DEBUG: unliked post not found in inbox or outbox')
2019-07-12 09:10:09 +00:00
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: liked post found in inbox. Now undoing.')
2020-04-03 16:27:34 +00:00
undoLikesCollectionEntry(recentPostsCache, baseDir, postFilename,
messageJson['object'],
messageJson['actor'], domain, debug)
2019-07-12 09:10:09 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveBookmark(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool) -> bool:
2019-11-17 14:01:49 +00:00
"""Receives a bookmark activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Bookmark':
2019-11-17 14:01:49 +00:00
return False
if not messageJson.get('actor'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no actor')
2019-11-17 14:01:49 +00:00
return False
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-11-17 14:01:49 +00:00
return False
if not isinstance(messageJson['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a string')
2019-11-17 14:01:49 +00:00
return False
if not messageJson.get('to'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
2019-11-17 14:01:49 +00:00
return False
if '/users/' not in messageJson['actor']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" missing from actor in ' +
messageJson['type'])
2019-11-17 14:01:49 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "statuses" missing from object in ' +
messageJson['type'])
2019-11-17 14:01:49 +00:00
return False
if domain not in handle.split('@')[1]:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: unrecognized domain ' + handle)
2020-03-22 21:16:02 +00:00
return False
2020-04-03 16:27:34 +00:00
domainFull = domain
2019-11-17 14:01:49 +00:00
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
domainFull = domain + ':' + str(port)
nickname = handle.split('@')[0]
if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
2019-11-17 14:01:49 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'bookmark actor should be the same as the handle sent to ' +
handle + ' != ' + messageJson['actor'])
2019-11-17 14:01:49 +00:00
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of bookmark - ' + handle)
2019-11-17 14:01:49 +00:00
# if this post in the outbox of the person?
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, nickname, domain, messageJson['object'])
2019-11-17 14:01:49 +00:00
if not postFilename:
if debug:
print('DEBUG: post not found in inbox or outbox')
print(messageJson['object'])
return True
if debug:
print('DEBUG: bookmarked post was found')
2020-04-03 16:27:34 +00:00
updateBookmarksCollection(recentPostsCache, baseDir, postFilename,
messageJson['object'],
messageJson['actor'], domain, debug)
2019-11-17 14:01:49 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveUndoBookmark(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool) -> bool:
2019-11-17 14:01:49 +00:00
"""Receives an undo bookmark activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Undo':
2019-11-17 14:01:49 +00:00
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('type'):
return False
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] != 'Bookmark':
2019-11-17 14:01:49 +00:00
return False
if not messageJson['object'].get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' like has no object')
2019-11-17 14:01:49 +00:00
return False
if not isinstance(messageJson['object']['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] +
' like object is not a string')
2019-11-17 14:01:49 +00:00
return False
if '/users/' not in messageJson['actor']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" missing from actor in ' +
messageJson['type'] + ' like')
2019-11-17 14:01:49 +00:00
return False
if '/statuses/' not in messageJson['object']['object']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "statuses" missing from like object in ' +
messageJson['type'])
2019-11-17 14:01:49 +00:00
return False
2020-04-03 16:27:34 +00:00
domainFull = domain
2019-11-17 14:01:49 +00:00
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
domainFull = domain + ':' + str(port)
nickname = handle.split('@')[0]
2019-11-17 14:01:49 +00:00
if domain not in handle.split('@')[1]:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: unrecognized bookmark domain ' + handle)
2020-03-22 21:16:02 +00:00
return False
2020-04-03 16:27:34 +00:00
if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
2019-11-17 14:01:49 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'bookmark actor should be the same as the handle sent to ' +
handle + ' != ' + messageJson['actor'])
2019-11-17 14:01:49 +00:00
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of bookmark undo - ' + handle)
2019-11-17 14:01:49 +00:00
# if this post in the outbox of the person?
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, nickname, domain,
messageJson['object']['object'])
2019-11-17 14:01:49 +00:00
if not postFilename:
if debug:
print('DEBUG: unbookmarked post not found in inbox or outbox')
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: bookmarked post found. Now undoing.')
2020-04-03 16:27:34 +00:00
undoBookmarksCollectionEntry(recentPostsCache, baseDir, postFilename,
messageJson['object'],
messageJson['actor'], domain, debug)
2019-11-17 14:01:49 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveDelete(session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool, allowDeletion: bool,
recentPostsCache: {}) -> bool:
2019-07-11 21:38:28 +00:00
"""Receives a Delete activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Delete':
2019-07-11 21:38:28 +00:00
return False
if not messageJson.get('actor'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no actor')
2019-07-11 21:38:28 +00:00
return False
2019-07-17 17:16:48 +00:00
if debug:
print('DEBUG: Delete activity arrived')
2019-07-11 21:38:28 +00:00
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-07-11 21:38:28 +00:00
return False
if not isinstance(messageJson['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a string')
2019-07-11 21:38:28 +00:00
return False
2020-04-03 16:27:34 +00:00
domainFull = domain
2019-08-12 18:02:29 +00:00
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
if ':' not in domain:
2020-04-03 16:27:34 +00:00
domainFull = domain + ':' + str(port)
deletePrefix = httpPrefix + '://' + domainFull + '/'
if (not allowDeletion and
(not messageJson['object'].startswith(deletePrefix) or
not messageJson['actor'].startswith(deletePrefix))):
2019-08-12 18:02:29 +00:00
if debug:
print('DEBUG: delete not permitted from other instances')
2020-03-22 21:16:02 +00:00
return False
2019-07-11 21:38:28 +00:00
if not messageJson.get('to'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
2019-07-11 21:38:28 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-11 21:38:28 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'"users" or "profile" missing from actor in ' +
messageJson['type'])
2019-07-11 21:38:28 +00:00
return False
if '/statuses/' not in messageJson['object']:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "statuses" missing from object in ' +
messageJson['type'])
2019-07-11 21:38:28 +00:00
return False
if messageJson['actor'] not in messageJson['object']:
if debug:
2020-03-22 21:16:02 +00:00
print('DEBUG: actor is not the owner of the post to be deleted')
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of like - ' + handle)
2019-07-11 21:38:28 +00:00
# if this post in the outbox of the person?
2020-08-23 11:13:35 +00:00
messageId = removeIdEnding(messageJson['object'])
2020-04-03 16:27:34 +00:00
removeModerationPostFromIndex(baseDir, messageId, debug)
postFilename = locatePost(baseDir, handle.split('@')[0],
handle.split('@')[1], messageId)
2019-07-11 21:38:28 +00:00
if not postFilename:
if debug:
print('DEBUG: delete post not found in inbox or outbox')
2019-07-17 17:16:48 +00:00
print(messageId)
return True
2020-04-03 16:27:34 +00:00
deletePost(baseDir, httpPrefix, handle.split('@')[0],
handle.split('@')[1], postFilename, debug,
recentPostsCache)
2019-07-11 21:38:28 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: post deleted - ' + postFilename)
2019-07-11 21:38:28 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveAnnounce(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, onionDomain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool, translate: {},
YTReplacementDomain: str) -> bool:
2019-07-12 09:41:57 +00:00
"""Receives an announce activity within the POST section of HTTPServer
2019-07-11 19:31:02 +00:00
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Announce':
2019-07-11 19:31:02 +00:00
return False
if '@' not in handle:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: bad handle ' + handle)
2020-03-22 21:16:02 +00:00
return False
2019-07-11 19:31:02 +00:00
if not messageJson.get('actor'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no actor')
2019-07-11 19:31:02 +00:00
return False
2019-07-16 22:57:45 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: receiving announce on ' + handle)
2019-07-11 19:31:02 +00:00
if not messageJson.get('object'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no object')
2019-07-11 19:31:02 +00:00
return False
if not isinstance(messageJson['object'], str):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' object is not a string')
2019-07-11 19:31:02 +00:00
return False
if not messageJson.get('to'):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
2019-07-11 19:31:02 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-11 19:31:02 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'"users" or "profile" missing from actor in ' +
messageJson['type'])
2019-09-09 09:41:31 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['object'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['object'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['object'] and \
'/profile/' not in messageJson['object']:
2019-09-09 09:41:31 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: ' +
'"users", "channel" or "profile" missing in ' +
messageJson['type'])
2019-07-11 19:31:02 +00:00
return False
2020-06-11 12:26:15 +00:00
prefixes = getProtocolPrefixes()
# is the domain of the announce actor blocked?
2020-06-11 12:04:42 +00:00
objectDomain = messageJson['object']
for prefix in prefixes:
objectDomain = objectDomain.replace(prefix, '')
if '/' in objectDomain:
2020-04-03 16:27:34 +00:00
objectDomain = objectDomain.split('/')[0]
if isBlockedDomain(baseDir, objectDomain):
if debug:
print('DEBUG: announced domain is blocked')
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of announce - ' + handle)
# is the announce actor blocked?
2020-04-03 16:27:34 +00:00
nickname = handle.split('@')[0]
actorNickname = getNicknameFromActor(messageJson['actor'])
actorDomain, actorPort = getDomainFromActor(messageJson['actor'])
if isBlocked(baseDir, nickname, domain, actorNickname, actorDomain):
print('Receive announce blocked for actor: ' +
actorNickname + '@' + actorDomain)
return False
# is this post in the outbox of the person?
postFilename = locatePost(baseDir, nickname, domain,
2020-04-03 16:27:34 +00:00
messageJson['object'])
2019-07-11 19:31:02 +00:00
if not postFilename:
if debug:
print('DEBUG: announce post not found in inbox or outbox')
print(messageJson['object'])
return True
2020-04-03 16:27:34 +00:00
updateAnnounceCollection(recentPostsCache, baseDir, postFilename,
messageJson['actor'], domain, debug)
2019-09-29 10:13:00 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Downloading announce post ' + messageJson['actor'] +
' -> ' + messageJson['object'])
postJsonObject = downloadAnnounce(session, baseDir, httpPrefix,
nickname, domain, messageJson,
__version__, translate,
YTReplacementDomain)
2019-09-30 19:13:14 +00:00
if postJsonObject:
2019-10-01 13:23:22 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Announce post downloaded for ' +
messageJson['actor'] + ' -> ' + messageJson['object'])
storeHashTags(baseDir, nickname, postJsonObject)
2019-09-30 19:13:14 +00:00
# Try to obtain the actor for this person
# so that their avatar can be shown
2020-04-03 16:27:34 +00:00
lookupActor = None
2019-10-01 14:11:15 +00:00
if postJsonObject.get('attributedTo'):
2020-08-06 16:21:46 +00:00
if isinstance(postJsonObject['attributedTo'], str):
lookupActor = postJsonObject['attributedTo']
2019-10-01 14:11:15 +00:00
else:
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('attributedTo'):
2020-08-06 16:21:46 +00:00
attrib = postJsonObject['object']['attributedTo']
if isinstance(attrib, str):
lookupActor = attrib
2019-09-30 19:13:14 +00:00
if lookupActor:
2019-10-17 22:26:47 +00:00
if '/users/' in lookupActor or \
2020-08-13 16:19:35 +00:00
'/accounts/' in lookupActor or \
2019-10-17 22:26:47 +00:00
'/channel/' in lookupActor or \
'/profile/' in lookupActor:
2019-10-01 13:23:22 +00:00
if '/statuses/' in lookupActor:
2020-04-03 16:27:34 +00:00
lookupActor = lookupActor.split('/statuses/')[0]
2019-10-01 12:35:39 +00:00
2019-10-01 12:50:06 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Obtaining actor for announce post ' +
lookupActor)
2019-10-01 13:23:22 +00:00
for tries in range(6):
2020-04-03 16:27:34 +00:00
pubKey = \
getPersonPubKey(baseDir, session, lookupActor,
personCache, debug,
__version__, httpPrefix,
domain, onionDomain)
2019-10-01 13:23:22 +00:00
if pubKey:
2020-04-03 16:27:34 +00:00
print('DEBUG: public key obtained for announce: ' +
lookupActor)
2019-10-01 13:23:22 +00:00
break
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Retry ' + str(tries + 1) +
' obtaining actor for ' + lookupActor)
2020-03-22 21:16:02 +00:00
time.sleep(5)
2019-07-11 19:31:02 +00:00
if debug:
2019-09-29 10:13:00 +00:00
print('DEBUG: announced/repeated post arrived in inbox')
2019-07-11 19:31:02 +00:00
return True
2020-04-03 16:27:34 +00:00
def receiveUndoAnnounce(recentPostsCache: {},
session, handle: str, isGroup: bool, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
debug: bool) -> bool:
2019-07-12 09:41:57 +00:00
"""Receives an undo announce activity within the POST section of HTTPServer
"""
2020-04-03 16:27:34 +00:00
if messageJson['type'] != 'Undo':
2019-07-12 09:41:57 +00:00
return False
if not messageJson.get('actor'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('object'):
return False
if not isinstance(messageJson['object']['object'], str):
return False
2020-04-03 16:27:34 +00:00
if messageJson['object']['type'] != 'Announce':
2020-03-22 21:16:02 +00:00
return False
2019-10-17 22:26:47 +00:00
if '/users/' not in messageJson['actor'] and \
2020-08-13 16:19:35 +00:00
'/accounts/' not in messageJson['actor'] and \
2019-10-17 22:26:47 +00:00
'/channel/' not in messageJson['actor'] and \
'/profile/' not in messageJson['actor']:
2019-07-12 09:41:57 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: "users" or "profile" missing from actor in ' +
messageJson['type'] + ' announce')
2019-07-12 09:41:57 +00:00
return False
2020-04-03 16:27:34 +00:00
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('DEBUG: unknown recipient of undo announce - ' + handle)
2019-07-12 09:41:57 +00:00
# if this post in the outbox of the person?
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, handle.split('@')[0],
handle.split('@')[1],
messageJson['object']['object'])
2019-07-12 09:41:57 +00:00
if not postFilename:
if debug:
print('DEBUG: undo announce post not found in inbox or outbox')
print(messageJson['object']['object'])
return True
if debug:
print('DEBUG: announced/repeated post to be undone found in inbox')
2020-04-03 16:27:34 +00:00
postJsonObject = loadJson(postFilename)
2019-10-22 11:55:06 +00:00
if postJsonObject:
2019-07-14 16:57:06 +00:00
if not postJsonObject.get('type'):
2020-04-03 16:27:34 +00:00
if postJsonObject['type'] != 'Announce':
if debug:
2020-04-03 16:27:34 +00:00
print("DEBUG: Attempt to undo something " +
"which isn't an announcement")
2020-03-22 21:16:02 +00:00
return False
2020-04-03 16:27:34 +00:00
undoAnnounceCollectionEntry(recentPostsCache, baseDir, postFilename,
messageJson['actor'], domain, debug)
if os.path.isfile(postFilename):
os.remove(postFilename)
2019-07-12 09:41:57 +00:00
return True
2020-04-03 16:27:34 +00:00
2020-08-21 18:32:16 +00:00
def jsonPostAllowsComments(postJsonObject: {}) -> bool:
"""Returns true if the given post allows comments/replies
"""
if 'commentsEnabled' in postJsonObject:
return postJsonObject['commentsEnabled']
if postJsonObject.get('object'):
if not isinstance(postJsonObject['object'], dict):
return False
if 'commentsEnabled' in postJsonObject['object']:
return postJsonObject['object']['commentsEnabled']
return True
def postAllowsComments(postFilename: str) -> bool:
"""Returns true if the given post allows comments/replies
"""
postJsonObject = loadJson(postFilename)
if not postJsonObject:
return False
return jsonPostAllowsComments(postJsonObject)
2020-04-03 16:27:34 +00:00
def populateReplies(baseDir: str, httpPrefix: str, domain: str,
messageJson: {}, maxReplies: int, debug: bool) -> bool:
2020-03-22 21:16:02 +00:00
"""Updates the list of replies for a post on this domain if
a reply to it arrives
"""
if not messageJson.get('id'):
return False
if not messageJson.get('object'):
return False
if not isinstance(messageJson['object'], dict):
return False
if not messageJson['object'].get('inReplyTo'):
return False
if not messageJson['object'].get('to'):
return False
2020-04-03 16:27:34 +00:00
replyTo = messageJson['object']['inReplyTo']
2020-08-28 14:45:07 +00:00
if not isinstance(replyTo, str):
return False
if debug:
print('DEBUG: post contains a reply')
# is this a reply to a post on this domain?
2020-04-03 16:27:34 +00:00
if not replyTo.startswith(httpPrefix + '://' + domain + '/'):
if debug:
print('DEBUG: post is a reply to another not on this domain')
2019-08-02 18:04:31 +00:00
print(replyTo)
2020-04-03 16:27:34 +00:00
print('Expected: ' + httpPrefix + '://' + domain + '/')
return False
2020-04-03 16:27:34 +00:00
replyToNickname = getNicknameFromActor(replyTo)
if not replyToNickname:
2020-04-03 16:27:34 +00:00
print('DEBUG: no nickname found for ' + replyTo)
return False
2020-04-03 16:27:34 +00:00
replyToDomain, replyToPort = getDomainFromActor(replyTo)
if not replyToDomain:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: no domain found for ' + replyTo)
return False
2020-04-03 16:27:34 +00:00
postFilename = locatePost(baseDir, replyToNickname,
replyToDomain, replyTo)
if not postFilename:
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: post may have expired - ' + replyTo)
2020-03-22 21:16:02 +00:00
return False
2020-08-21 18:32:16 +00:00
if not postAllowsComments(postFilename):
if debug:
print('DEBUG: post does not allow comments - ' + replyTo)
return False
2019-07-13 19:28:14 +00:00
# populate a text file containing the ids of replies
2020-04-03 16:27:34 +00:00
postRepliesFilename = postFilename.replace('.json', '.replies')
2020-08-23 11:13:35 +00:00
messageId = removeIdEnding(messageJson['id'])
2019-07-13 19:28:14 +00:00
if os.path.isfile(postRepliesFilename):
2020-04-03 16:27:34 +00:00
numLines = sum(1 for line in open(postRepliesFilename))
if numLines > maxReplies:
2019-07-13 21:00:12 +00:00
return False
2019-07-13 19:28:14 +00:00
if messageId not in open(postRepliesFilename).read():
2020-08-20 11:34:39 +00:00
repliesFile = open(postRepliesFilename, 'a+')
2020-04-03 16:27:34 +00:00
repliesFile.write(messageId + '\n')
2019-07-13 19:28:14 +00:00
repliesFile.close()
else:
2020-08-29 11:14:19 +00:00
repliesFile = open(postRepliesFilename, 'w+')
2020-04-03 16:27:34 +00:00
repliesFile.write(messageId + '\n')
2019-07-13 19:28:14 +00:00
repliesFile.close()
return True
2019-09-30 09:43:46 +00:00
2020-04-03 16:27:34 +00:00
2019-09-30 10:15:20 +00:00
def estimateNumberOfMentions(content: str) -> int:
"""Returns a rough estimate of the number of mentions
"""
2020-04-03 16:27:34 +00:00
return int(content.count('@') / 2)
2019-11-16 14:49:21 +00:00
def estimateNumberOfEmoji(content: str) -> int:
"""Returns a rough estimate of the number of emoji
"""
2020-04-03 16:27:34 +00:00
return int(content.count(':') / 2)
2019-11-16 14:49:21 +00:00
2020-04-03 16:27:34 +00:00
def validPostContent(baseDir: str, nickname: str, domain: str,
messageJson: {}, maxMentions: int, maxEmoji: int) -> bool:
2019-09-30 09:43:46 +00:00
"""Is the content of a received post valid?
2019-09-30 10:15:20 +00:00
Check for bad html
Check for hellthreads
Check number of tags is reasonable
2019-09-30 09:43:46 +00:00
"""
if not messageJson.get('object'):
return True
if not isinstance(messageJson['object'], dict):
return True
if not messageJson['object'].get('content'):
return True
2019-11-29 22:45:56 +00:00
if not messageJson['object'].get('published'):
return False
if 'T' not in messageJson['object']['published']:
return False
if 'Z' not in messageJson['object']['published']:
return False
2020-07-10 14:15:01 +00:00
2020-08-25 19:45:15 +00:00
if messageJson['object'].get('summary'):
summary = messageJson['object']['summary']
if not isinstance(summary, str):
print('WARN: content warning is not a string')
return False
if summary != validContentWarning(summary):
print('WARN: invalid content warning ' + summary)
return False
2020-05-02 11:08:38 +00:00
if isGitPatch(baseDir, nickname, domain,
2020-05-03 10:56:29 +00:00
messageJson['object']['type'],
2020-05-02 11:08:38 +00:00
messageJson['object']['summary'],
messageJson['object']['content']):
return True
2020-07-10 14:15:01 +00:00
if dangerousMarkup(messageJson['object']['content']):
if messageJson['object'].get('id'):
print('REJECT ARBITRARY HTML: ' + messageJson['object']['id'])
print('REJECT ARBITRARY HTML: bad string in post - ' +
messageJson['object']['content'])
return False
# check (rough) number of mentions
2020-04-03 16:27:34 +00:00
mentionsEst = estimateNumberOfMentions(messageJson['object']['content'])
if mentionsEst > maxMentions:
2019-09-30 10:37:34 +00:00
if messageJson['object'].get('id'):
2020-04-03 16:27:34 +00:00
print('REJECT HELLTHREAD: ' + messageJson['object']['id'])
print('REJECT HELLTHREAD: Too many mentions in post - ' +
messageJson['object']['content'])
2019-11-16 14:49:21 +00:00
return False
2020-04-03 16:27:34 +00:00
if estimateNumberOfEmoji(messageJson['object']['content']) > maxEmoji:
2019-11-16 14:49:21 +00:00
if messageJson['object'].get('id'):
2020-04-03 16:27:34 +00:00
print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id'])
print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' +
messageJson['object']['content'])
2019-09-30 10:15:20 +00:00
return False
# check number of tags
if messageJson['object'].get('tag'):
if not isinstance(messageJson['object']['tag'], list):
2020-04-03 16:27:34 +00:00
messageJson['object']['tag'] = []
else:
2020-04-03 16:27:34 +00:00
if len(messageJson['object']['tag']) > int(maxMentions * 2):
2019-09-30 10:37:34 +00:00
if messageJson['object'].get('id'):
2020-04-03 16:27:34 +00:00
print('REJECT: ' + messageJson['object']['id'])
print('REJECT: Too many tags in post - ' +
messageJson['object']['tag'])
return False
2020-02-05 17:29:38 +00:00
# check for filtered content
2020-04-03 16:27:34 +00:00
if isFiltered(baseDir, nickname, domain,
messageJson['object']['content']):
2020-02-05 17:29:38 +00:00
print('REJECT: content filtered')
return False
2020-08-21 18:32:16 +00:00
if messageJson['object'].get('inReplyTo'):
if isinstance(messageJson['object']['inReplyTo'], str):
originalPostId = messageJson['object']['inReplyTo']
postPostFilename = locatePost(baseDir, nickname, domain,
originalPostId)
if postPostFilename:
if not postAllowsComments(postPostFilename):
print('REJECT: reply to post which does not ' +
'allow comments: ' + originalPostId)
return False
2019-09-30 09:43:46 +00:00
print('ACCEPT: post content is valid')
return True
2020-04-03 16:27:34 +00:00
def obtainAvatarForReplyPost(session, baseDir: str, httpPrefix: str,
domain: str, onionDomain: str, personCache: {},
postJsonObject: {}, debug: bool) -> None:
"""Tries to obtain the actor for the person being replied to
so that their avatar can later be shown
"""
2019-09-30 19:39:48 +00:00
if not postJsonObject.get('object'):
return
2020-03-22 21:16:02 +00:00
2019-09-30 19:39:48 +00:00
if not isinstance(postJsonObject['object'], dict):
return
if not postJsonObject['object'].get('inReplyTo'):
return
2020-04-03 16:27:34 +00:00
lookupActor = postJsonObject['object']['inReplyTo']
2019-10-21 12:49:16 +00:00
if not lookupActor:
return
2020-08-28 14:45:07 +00:00
if not isinstance(lookupActor, str):
return
2020-04-03 16:27:34 +00:00
if not ('/users/' in lookupActor or
2020-08-13 16:19:35 +00:00
'/accounts/' in lookupActor or
2020-04-03 16:27:34 +00:00
'/channel/' in lookupActor or
2019-10-21 12:49:16 +00:00
'/profile/' in lookupActor):
return
if '/statuses/' in lookupActor:
2020-04-03 16:27:34 +00:00
lookupActor = lookupActor.split('/statuses/')[0]
2020-03-22 21:16:02 +00:00
2019-10-21 12:49:16 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Obtaining actor for reply post ' + lookupActor)
2019-10-01 13:23:22 +00:00
2019-10-21 12:49:16 +00:00
for tries in range(6):
2020-04-03 16:27:34 +00:00
pubKey = \
getPersonPubKey(baseDir, session, lookupActor,
personCache, debug,
__version__, httpPrefix,
domain, onionDomain)
2019-10-21 12:49:16 +00:00
if pubKey:
2020-04-03 16:27:34 +00:00
print('DEBUG: public key obtained for reply: ' + lookupActor)
2019-10-21 12:49:16 +00:00
break
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Retry ' + str(tries + 1) +
' obtaining actor for ' + lookupActor)
2020-03-22 21:16:02 +00:00
time.sleep(5)
2020-04-03 16:27:34 +00:00
def dmNotify(baseDir: str, handle: str, url: str) -> None:
"""Creates a notification that a new DM has arrived
"""
2020-04-03 16:27:34 +00:00
accountDir = baseDir + '/accounts/' + handle
if not os.path.isdir(accountDir):
return
2020-04-03 16:27:34 +00:00
dmFile = accountDir + '/.newDM'
if not os.path.isfile(dmFile):
2020-07-12 20:04:58 +00:00
with open(dmFile, 'w+') as fp:
2019-10-06 15:07:40 +00:00
fp.write(url)
2020-04-03 16:27:34 +00:00
2020-07-13 13:36:45 +00:00
def alreadyLiked(baseDir: str, nickname: str, domain: str,
postUrl: str, likerActor: str) -> bool:
"""Is the given post already liked by the given handle?
"""
postFilename = \
locatePost(baseDir, nickname, domain, postUrl)
if not postFilename:
return False
postJsonObject = loadJson(postFilename, 1)
if not postJsonObject:
return False
if not postJsonObject.get('object'):
return False
if not isinstance(postJsonObject['object'], dict):
return False
if not postJsonObject['object'].get('likes'):
return False
if not postJsonObject['object']['likes'].get('items'):
return False
for like in postJsonObject['object']['likes']['items']:
if not like.get('type'):
continue
if not like.get('actor'):
continue
if like['type'] != 'Like':
continue
if like['actor'] == likerActor:
return True
return False
def likeNotify(baseDir: str, domain: str, onionDomain: str,
handle: str, actor: str, url: str) -> None:
2020-07-08 19:49:15 +00:00
"""Creates a notification that a like has arrived
"""
# This is not you liking your own post
if actor in url:
return
# check that the liked post was by this handle
nickname = handle.split('@')[0]
if '/' + domain + '/users/' + nickname not in url:
if not onionDomain:
return
if '/' + onionDomain + '/users/' + nickname not in url:
return
2020-07-08 19:49:15 +00:00
accountDir = baseDir + '/accounts/' + handle
2020-08-27 09:35:26 +00:00
# are like notifications enabled?
2020-08-27 09:19:32 +00:00
notifyLikesEnabledFilename = accountDir + '/.notifyLikes'
if not os.path.isfile(notifyLikesEnabledFilename):
2020-07-08 19:49:15 +00:00
return
2020-08-27 09:19:32 +00:00
2020-07-08 19:49:15 +00:00
likeFile = accountDir + '/.newLike'
2020-07-08 21:41:48 +00:00
if os.path.isfile(likeFile):
2020-07-08 21:52:18 +00:00
if '##sent##' not in open(likeFile).read():
return
2020-07-08 19:49:15 +00:00
likerNickname = getNicknameFromActor(actor)
likerDomain, likerPort = getDomainFromActor(actor)
2020-07-08 21:04:19 +00:00
if likerNickname and likerDomain:
likerHandle = likerNickname + '@' + likerDomain
else:
2020-07-08 21:18:50 +00:00
print('likeNotify likerHandle: ' +
str(likerNickname) + '@' + str(likerDomain))
2020-07-08 21:04:19 +00:00
likerHandle = actor
2020-07-08 19:49:15 +00:00
if likerHandle != handle:
2020-07-13 19:42:30 +00:00
likeStr = likerHandle + ' ' + url + '?likedBy=' + actor
2020-07-08 22:17:21 +00:00
prevLikeFile = accountDir + '/.prevLike'
# was there a previous like notification?
if os.path.isfile(prevLikeFile):
# is it the same as the current notification ?
2020-07-13 09:34:04 +00:00
with open(prevLikeFile, 'r') as fp:
prevLikeStr = fp.read()
2020-07-08 22:17:21 +00:00
if prevLikeStr == likeStr:
return
2020-07-12 20:04:58 +00:00
try:
with open(prevLikeFile, 'w+') as fp:
fp.write(likeStr)
except BaseException:
print('ERROR: unable to save previous like notification ' +
prevLikeFile)
2020-07-12 20:04:58 +00:00
pass
try:
with open(likeFile, 'w+') as fp:
fp.write(likeStr)
except BaseException:
print('ERROR: unable to write like notification file ' +
likeFile)
2020-07-12 20:04:58 +00:00
pass
2020-07-08 19:49:15 +00:00
2020-04-03 16:27:34 +00:00
def replyNotify(baseDir: str, handle: str, url: str) -> None:
"""Creates a notification that a new reply has arrived
"""
2020-04-03 16:27:34 +00:00
accountDir = baseDir + '/accounts/' + handle
if not os.path.isdir(accountDir):
return
2020-04-03 16:27:34 +00:00
replyFile = accountDir + '/.newReply'
if not os.path.isfile(replyFile):
2020-07-12 20:04:58 +00:00
with open(replyFile, 'w+') as fp:
2019-10-06 15:11:10 +00:00
fp.write(url)
2020-04-03 16:27:34 +00:00
def gitPatchNotify(baseDir: str, handle: str,
subject: str, content: str,
fromNickname: str, fromDomain: str) -> None:
"""Creates a notification that a new git patch has arrived
"""
accountDir = baseDir + '/accounts/' + handle
if not os.path.isdir(accountDir):
return
patchFile = accountDir + '/.newPatch'
subject = subject.replace('[PATCH]', '').strip()
handle = '@' + fromNickname + '@' + fromDomain
2020-07-12 20:04:58 +00:00
with open(patchFile, 'w+') as fp:
fp.write('git ' + handle + ' ' + subject)
2020-05-02 17:16:24 +00:00
2020-04-03 16:27:34 +00:00
def groupHandle(baseDir: str, handle: str) -> bool:
2019-10-04 12:22:56 +00:00
"""Is the given account handle a group?
"""
2020-04-03 16:27:34 +00:00
actorFile = baseDir + '/accounts/' + handle + '.json'
2019-10-04 12:22:56 +00:00
if not os.path.isfile(actorFile):
return False
2020-04-03 16:27:34 +00:00
actorJson = loadJson(actorFile)
2019-10-04 12:22:56 +00:00
if not actorJson:
return False
2020-04-03 16:27:34 +00:00
return actorJson['type'] == 'Group'
2019-10-04 12:22:56 +00:00
2020-04-03 16:27:34 +00:00
def getGroupName(baseDir: str, handle: str) -> str:
2019-10-04 13:39:41 +00:00
"""Returns the preferred name of a group
"""
2020-04-03 16:27:34 +00:00
actorFile = baseDir + '/accounts/' + handle + '.json'
2019-10-04 13:39:41 +00:00
if not os.path.isfile(actorFile):
return False
2020-04-03 16:27:34 +00:00
actorJson = loadJson(actorFile)
2019-10-04 13:39:41 +00:00
if not actorJson:
return 'Group'
return actorJson['name']
2020-04-03 16:27:34 +00:00
def sendToGroupMembers(session, baseDir: str, handle: str, port: int,
postJsonObject: {},
httpPrefix: str, federationList: [],
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, debug: bool) -> None:
2019-10-04 12:22:56 +00:00
"""When a post arrives for a group send it out to the group members
"""
2020-04-03 16:27:34 +00:00
followersFile = baseDir + '/accounts/' + handle + '/followers.txt'
2019-10-04 12:22:56 +00:00
if not os.path.isfile(followersFile):
return
2019-10-04 13:31:30 +00:00
if not postJsonObject.get('object'):
return
2020-04-03 16:27:34 +00:00
nickname = handle.split('@')[0]
# groupname = getGroupName(baseDir, handle)
domain = handle.split('@')[1]
domainFull = domain
2019-10-04 14:02:11 +00:00
if ':' not in domain:
if port:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
domain = domain + ':' + str(port)
2019-10-04 14:02:11 +00:00
# set sender
2020-04-03 16:27:34 +00:00
cc = ''
sendingActor = postJsonObject['actor']
sendingActorNickname = getNicknameFromActor(sendingActor)
sendingActorDomain, sendingActorPort = \
getDomainFromActor(sendingActor)
sendingActorDomainFull = sendingActorDomain
if ':' in sendingActorDomain:
if sendingActorPort:
2020-04-03 16:27:34 +00:00
if sendingActorPort != 80 and sendingActorPort != 443:
sendingActorDomainFull = \
sendingActorDomain + ':' + str(sendingActorPort)
senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull
if not postJsonObject['object']['content'].startswith(senderStr):
2020-04-03 16:27:34 +00:00
postJsonObject['object']['content'] = \
senderStr + ' ' + postJsonObject['object']['content']
# add mention to tag list
2019-10-04 14:38:18 +00:00
if not postJsonObject['object']['tag']:
2020-04-03 16:27:34 +00:00
postJsonObject['object']['tag'] = []
2019-10-04 15:17:48 +00:00
# check if the mention already exists
2020-04-03 16:27:34 +00:00
mentionExists = False
2019-10-04 15:17:48 +00:00
for mention in postJsonObject['object']['tag']:
2020-04-03 16:27:34 +00:00
if mention['type'] == 'Mention':
2019-10-04 15:17:48 +00:00
if mention.get('href'):
2020-04-03 16:27:34 +00:00
if mention['href'] == sendingActor:
mentionExists = True
2019-10-04 15:17:48 +00:00
if not mentionExists:
# add the mention of the original sender
postJsonObject['object']['tag'].append({
'href': sendingActor,
'name': senderStr,
'type': 'Mention'
})
2020-04-03 16:27:34 +00:00
postJsonObject['actor'] = \
httpPrefix + '://' + domainFull + '/users/' + nickname
postJsonObject['to'] = \
[postJsonObject['actor'] + '/followers']
postJsonObject['cc'] = [cc]
postJsonObject['object']['to'] = postJsonObject['to']
postJsonObject['object']['cc'] = [cc]
2019-10-04 14:09:48 +00:00
# set subject
if not postJsonObject['object'].get('summary'):
2020-04-03 16:27:34 +00:00
postJsonObject['object']['summary'] = 'General Discussion'
2019-10-04 12:22:56 +00:00
if ':' in domain:
2020-04-03 16:27:34 +00:00
domain = domain.split(':')[0]
2019-10-04 12:22:56 +00:00
with open(followersFile, 'r') as groupMembers:
for memberHandle in groupMembers:
2020-04-03 16:27:34 +00:00
if memberHandle != handle:
memberNickname = memberHandle.split('@')[0]
memberDomain = memberHandle.split('@')[1]
memberPort = port
2019-10-04 12:22:56 +00:00
if ':' in memberDomain:
2020-04-03 16:27:34 +00:00
memberPortStr = memberDomain.split(':')[1]
2019-10-04 12:22:56 +00:00
if memberPortStr.isdigit():
2020-04-03 16:27:34 +00:00
memberPort = int(memberPortStr)
memberDomain = memberDomain.split(':')[0]
sendSignedJson(postJsonObject, session, baseDir,
nickname, domain, port,
memberNickname, memberDomain, memberPort, cc,
httpPrefix, False, False, federationList,
sendThreads, postLog, cachedWebfingers,
personCache, debug, __version__)
def inboxUpdateCalendar(baseDir: str, handle: str, postJsonObject: {}) -> None:
2019-10-11 12:31:06 +00:00
"""Detects whether the tag list on a post contains calendar events
and if so saves the post id to a file in the calendar directory
for the account
"""
if not postJsonObject.get('actor'):
return
2019-10-11 12:31:06 +00:00
if not postJsonObject.get('object'):
return
if not isinstance(postJsonObject['object'], dict):
return
if not postJsonObject['object'].get('tag'):
return
if not isinstance(postJsonObject['object']['tag'], list):
return
actor = postJsonObject['actor']
actorNickname = getNicknameFromActor(actor)
actorDomain, actorPort = getDomainFromActor(actor)
2020-07-11 21:01:08 +00:00
handleNickname = handle.split('@')[0]
handleDomain = handle.split('@')[1]
if not receivingCalendarEvents(baseDir,
2020-07-11 21:01:08 +00:00
handleNickname, handleDomain,
actorNickname, actorDomain):
return
2020-08-13 09:37:11 +00:00
2020-08-23 11:13:35 +00:00
postId = removeIdEnding(postJsonObject['id']).replace('/', '#')
2020-08-13 11:58:05 +00:00
2020-08-13 09:37:11 +00:00
# look for events within the tags list
2019-10-11 12:31:06 +00:00
for tagDict in postJsonObject['object']['tag']:
2020-07-11 22:36:52 +00:00
if not tagDict.get('type'):
continue
2020-04-03 16:27:34 +00:00
if tagDict['type'] != 'Event':
2019-10-11 12:31:06 +00:00
continue
2019-10-11 16:16:56 +00:00
if not tagDict.get('startTime'):
2019-10-11 12:31:06 +00:00
continue
2020-08-20 16:51:48 +00:00
saveEventPost(baseDir, handle, postId, tagDict)
2020-04-03 16:27:34 +00:00
def inboxUpdateIndex(boxname: str, baseDir: str, handle: str,
destinationFilename: str, debug: bool) -> bool:
2019-10-20 10:25:38 +00:00
"""Updates the index of received posts
The new entry is added to the top of the file
"""
2020-04-03 16:27:34 +00:00
indexFilename = baseDir + '/accounts/' + handle + '/' + boxname + '.index'
2019-10-20 10:40:09 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Updating index ' + indexFilename)
2019-11-18 13:16:21 +00:00
2020-04-03 16:27:34 +00:00
if '/' + boxname + '/' in destinationFilename:
destinationFilename = destinationFilename.split('/' + boxname + '/')[1]
2019-11-18 13:16:21 +00:00
# remove the path
if '/' in destinationFilename:
2020-04-03 16:27:34 +00:00
destinationFilename = destinationFilename.split('/')[-1]
2019-11-18 13:16:21 +00:00
2019-10-20 10:45:12 +00:00
if os.path.isfile(indexFilename):
2019-10-20 12:43:59 +00:00
try:
with open(indexFilename, 'r+') as indexFile:
2020-04-03 16:27:34 +00:00
content = indexFile.read()
2019-10-20 12:43:59 +00:00
indexFile.seek(0, 0)
2020-04-03 16:27:34 +00:00
indexFile.write(destinationFilename + '\n' + content)
2019-10-20 12:43:59 +00:00
return True
except Exception as e:
2020-04-03 16:27:34 +00:00
print('WARN: Failed to write entry to index ' + str(e))
2019-10-20 10:45:12 +00:00
else:
2019-10-20 12:43:59 +00:00
try:
2020-04-03 16:27:34 +00:00
indexFile = open(indexFilename, 'w+')
2019-10-20 12:43:59 +00:00
if indexFile:
2020-04-03 16:27:34 +00:00
indexFile.write(destinationFilename + '\n')
2019-10-20 12:43:59 +00:00
indexFile.close()
except Exception as e:
2020-04-03 16:27:34 +00:00
print('WARN: Failed to write initial entry to index ' + str(e))
2019-10-20 10:45:12 +00:00
2019-10-20 10:35:13 +00:00
return False
2019-10-20 10:25:38 +00:00
2020-04-03 16:27:34 +00:00
def inboxAfterCapabilities(recentPostsCache: {}, maxRecentPosts: int,
session, keyId: str, handle: str, messageJson: {},
baseDir: str, httpPrefix: str, sendThreads: [],
postLog: [], cachedWebfingers: {}, personCache: {},
2020-06-03 20:21:44 +00:00
queue: [], domain: str,
onionDomain: str, i2pDomain: str,
2020-06-09 11:03:59 +00:00
port: int, proxyType: str,
2020-04-03 16:27:34 +00:00
federationList: [], ocapAlways: bool, debug: bool,
acceptedCaps: [],
queueFilename: str, destinationFilename: str,
maxReplies: int, allowDeletion: bool,
maxMentions: int, maxEmoji: int, translate: {},
unitTest: bool, YTReplacementDomain: str) -> bool:
""" Anything which needs to be done after capabilities checks have passed
"""
2020-04-03 16:27:34 +00:00
actor = keyId
2019-09-29 10:41:21 +00:00
if '#' in actor:
2020-04-03 16:27:34 +00:00
actor = keyId.split('#')[0]
isGroup = groupHandle(baseDir, handle)
if receiveLike(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
onionDomain,
2020-04-03 16:27:34 +00:00
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
2019-07-10 12:40:31 +00:00
debug):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Like accepted from ' + actor)
2019-07-10 12:40:31 +00:00
return False
2020-04-03 16:27:34 +00:00
if receiveUndoLike(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
2019-07-12 09:10:09 +00:00
debug):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Undo like accepted from ' + actor)
2019-07-12 09:10:09 +00:00
return False
2020-04-03 16:27:34 +00:00
if receiveBookmark(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
2019-11-17 14:01:49 +00:00
debug):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Bookmark accepted from ' + actor)
2019-11-17 14:01:49 +00:00
return False
2020-04-03 16:27:34 +00:00
if receiveUndoBookmark(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
2019-11-17 14:01:49 +00:00
debug):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Undo bookmark accepted from ' + actor)
2019-11-17 14:01:49 +00:00
return False
2020-03-22 21:16:02 +00:00
2020-06-16 20:29:17 +00:00
# labelAccusatoryPost(messageJson, translate)
2020-06-12 11:50:49 +00:00
2020-04-03 16:27:34 +00:00
if receiveAnnounce(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, onionDomain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
debug, translate,
YTReplacementDomain):
2019-07-11 19:31:02 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Announce accepted from ' + actor)
if receiveUndoAnnounce(recentPostsCache,
session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
2019-07-12 09:41:57 +00:00
debug):
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Undo announce accepted from ' + actor)
2019-07-12 11:35:03 +00:00
return False
2019-07-12 09:41:57 +00:00
2020-04-03 16:27:34 +00:00
if receiveDelete(session, handle, isGroup,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
messageJson,
federationList,
debug, allowDeletion,
recentPostsCache):
2019-08-12 18:02:29 +00:00
if debug:
2020-04-03 16:27:34 +00:00
print('DEBUG: Delete accepted from ' + actor)
2019-08-12 18:02:29 +00:00
return False
2019-07-10 13:32:47 +00:00
if debug:
print('DEBUG: object capabilities passed')
2020-04-03 16:27:34 +00:00
print('copy queue file from ' + queueFilename +
' to ' + destinationFilename)
2019-08-16 22:04:45 +00:00
if os.path.isfile(destinationFilename):
return True
2019-10-04 09:58:02 +00:00
if messageJson.get('postNickname'):
2020-04-03 16:27:34 +00:00
postJsonObject = messageJson['post']
else:
2020-04-03 16:27:34 +00:00
postJsonObject = messageJson
2019-10-04 12:22:56 +00:00
2020-04-03 16:27:34 +00:00
nickname = handle.split('@')[0]
if validPostContent(baseDir, nickname, domain,
postJsonObject, maxMentions, maxEmoji):
2020-05-02 13:17:02 +00:00
2020-08-23 14:45:58 +00:00
if postJsonObject.get('object'):
jsonObj = postJsonObject['object']
if not isinstance(jsonObj, dict):
jsonObj = None
else:
jsonObj = postJsonObject
2020-05-02 13:17:02 +00:00
# check for incoming git patches
2020-08-23 14:45:58 +00:00
if jsonObj:
if jsonObj.get('content') and \
jsonObj.get('summary') and \
jsonObj.get('attributedTo'):
attributedTo = jsonObj['attributedTo']
2020-08-06 16:21:46 +00:00
if isinstance(attributedTo, str):
fromNickname = getNicknameFromActor(attributedTo)
fromDomain, fromPort = getDomainFromActor(attributedTo)
if fromPort:
if fromPort != 80 and fromPort != 443:
fromDomain += ':' + str(fromPort)
if receiveGitPatch(baseDir, nickname, domain,
2020-08-23 14:45:58 +00:00
jsonObj['type'],
jsonObj['summary'],
jsonObj['content'],
2020-08-06 16:21:46 +00:00
fromNickname, fromDomain):
gitPatchNotify(baseDir, handle,
2020-08-23 14:45:58 +00:00
jsonObj['summary'],
jsonObj['content'],
2020-08-06 16:21:46 +00:00
fromNickname, fromDomain)
2020-08-23 14:45:58 +00:00
elif '[PATCH]' in jsonObj['content']:
2020-08-06 16:21:46 +00:00
print('WARN: git patch not accepted - ' +
2020-08-23 14:45:58 +00:00
jsonObj['summary'])
2020-08-06 16:21:46 +00:00
return False
2020-05-02 11:08:38 +00:00
# replace YouTube links, so they get less tracking data
replaceYouTube(postJsonObject, YTReplacementDomain)
2019-10-22 20:07:12 +00:00
# list of indexes to be updated
2020-04-03 16:27:34 +00:00
updateIndexList = ['inbox']
populateReplies(baseDir, httpPrefix, domain, postJsonObject,
maxReplies, debug)
2019-11-29 19:22:11 +00:00
# if this is a reply to a question then update the votes
2020-04-03 16:27:34 +00:00
questionJson = questionUpdateVotes(baseDir, nickname, domain,
postJsonObject)
if questionJson:
# Is this a question created by this instance?
2020-04-03 16:27:34 +00:00
idPrefix = httpPrefix + '://' + domain
if questionJson['object']['id'].startswith(idPrefix):
# if the votes on a question have changed then
# send out an update
questionJson['type'] = 'Update'
sendToFollowersThread(session, baseDir,
2020-06-03 20:21:44 +00:00
nickname, domain,
onionDomain, i2pDomain, port,
2020-04-03 16:27:34 +00:00
httpPrefix, federationList,
sendThreads, postLog,
cachedWebfingers, personCache,
postJsonObject, debug,
__version__)
2019-11-29 19:22:11 +00:00
2020-08-27 17:40:09 +00:00
isReplyToMutedPost = False
2019-10-04 12:22:56 +00:00
if not isGroup:
# create a DM notification file if needed
postIsDM = isDM(postJsonObject)
if postIsDM:
2020-04-03 16:27:34 +00:00
if nickname != 'inbox':
followDMsFilename = \
baseDir + '/accounts/' + \
nickname + '@' + domain + '/.followDMs'
if os.path.isfile(followDMsFilename):
2020-04-03 16:27:34 +00:00
followingFilename = \
baseDir + '/accounts/' + \
nickname + '@' + domain + '/following.txt'
if not postJsonObject.get('actor'):
return False
2020-04-03 16:27:34 +00:00
sendingActor = postJsonObject['actor']
sendingActorNickname = \
getNicknameFromActor(sendingActor)
sendingActorDomain, sendingActorPort = \
getDomainFromActor(sendingActor)
if sendingActorNickname and sendingActorDomain:
2020-04-03 16:27:34 +00:00
sendH = \
sendingActorNickname + '@' + sendingActorDomain
if sendH != nickname + '@' + domain:
if sendH not in open(followingFilename).read():
print(nickname + '@' + domain +
' cannot receive DM from ' + sendH +
' because they do not follow them')
return False
else:
return False
2019-10-22 20:07:12 +00:00
# dm index will be updated
updateIndexList.append('dm')
2020-04-03 16:27:34 +00:00
dmNotify(baseDir, handle,
httpPrefix + '://' + domain + '/users/' +
nickname + '/dm')
2019-10-04 12:22:56 +00:00
# get the actor being replied to
2020-04-03 16:27:34 +00:00
domainFull = domain
2019-10-04 12:22:56 +00:00
if port:
if ':' not in domain:
2020-04-03 16:27:34 +00:00
if port != 80 and port != 443:
domainFull = domainFull + ':' + str(port)
actor = httpPrefix + '://' + domainFull + \
'/users/' + handle.split('@')[0]
2019-10-04 12:22:56 +00:00
# create a reply notification file if needed
if not postIsDM and isReply(postJsonObject, actor):
2020-04-03 16:27:34 +00:00
if nickname != 'inbox':
2019-10-22 20:07:12 +00:00
# replies index will be updated
updateIndexList.append('tlreplies')
2020-08-28 14:28:59 +00:00
inReplyTo = postJsonObject['object']['inReplyTo']
if inReplyTo:
if isinstance(inReplyTo, str):
if not isMuted(baseDir, nickname, domain,
inReplyTo):
replyNotify(baseDir, handle,
httpPrefix + '://' + domain +
'/users/' + nickname +
'/tlreplies')
else:
isReplyToMutedPost = True
2019-10-04 10:00:57 +00:00
2020-04-03 16:27:34 +00:00
if isImageMedia(session, baseDir, httpPrefix,
2020-06-12 11:50:49 +00:00
nickname, domain, postJsonObject,
translate, YTReplacementDomain):
2019-10-22 20:30:43 +00:00
# media index will be updated
updateIndexList.append('tlmedia')
2020-02-24 14:39:25 +00:00
if isBlogPost(postJsonObject):
# blogs index will be updated
updateIndexList.append('tlblogs')
2020-08-23 11:13:35 +00:00
elif isEventPost(postJsonObject):
# events index will be updated
updateIndexList.append('tlevents')
2019-10-22 20:30:43 +00:00
2019-10-04 10:00:57 +00:00
# get the avatar for a reply/announce
2020-04-03 16:27:34 +00:00
obtainAvatarForReplyPost(session, baseDir,
httpPrefix, domain, onionDomain,
personCache, postJsonObject, debug)
2019-10-04 10:00:57 +00:00
# save the post to file
2020-04-03 16:27:34 +00:00
if saveJson(postJsonObject, destinationFilename):
2020-08-27 17:40:09 +00:00
# If this is a reply to a muted post then also mute it.
# This enables you to ignore a threat that's getting boring
if isReplyToMutedPost:
print('MUTE REPLY: ' + destinationFilename)
2020-08-29 11:14:19 +00:00
muteFile = open(destinationFilename + '.muted', 'w+')
2020-08-27 17:40:09 +00:00
if muteFile:
muteFile.write('\n')
muteFile.close()
2019-10-22 20:07:12 +00:00
# update the indexes for different timelines
for boxname in updateIndexList:
2020-04-03 16:27:34 +00:00
if not inboxUpdateIndex(boxname, baseDir, handle,
destinationFilename, debug):
print('ERROR: unable to update ' + boxname + ' index')
else:
if not unitTest:
2020-08-26 11:19:32 +00:00
if debug:
print('Saving inbox post as html to cache')
htmlCacheStartTime = time.time()
inboxStorePostToHtmlCache(recentPostsCache,
maxRecentPosts,
translate, baseDir,
httpPrefix,
session, cachedWebfingers,
personCache,
handle.split('@')[0],
domain, port,
postJsonObject,
allowDeletion,
boxname)
2020-08-26 11:19:32 +00:00
if debug:
timeDiff = \
str(int((time.time() - htmlCacheStartTime) *
1000))
print('Saved ' + boxname +
' post as html to cache in ' +
timeDiff + ' mS')
2019-10-20 10:25:38 +00:00
2020-04-03 16:27:34 +00:00
inboxUpdateCalendar(baseDir, handle, postJsonObject)
2019-10-19 18:08:47 +00:00
2020-04-03 16:27:34 +00:00
storeHashTags(baseDir, handle.split('@')[0], postJsonObject)
2019-12-12 17:34:31 +00:00
2019-10-19 13:00:46 +00:00
# send the post out to group members
if isGroup:
2020-04-03 16:27:34 +00:00
sendToGroupMembers(session, baseDir, handle, port,
postJsonObject,
httpPrefix, federationList, sendThreads,
postLog, cachedWebfingers, personCache,
debug)
2019-10-04 12:22:56 +00:00
2019-10-04 10:00:57 +00:00
# if the post wasn't saved
2019-08-17 12:26:09 +00:00
if not os.path.isfile(destinationFilename):
return False
return True
2020-04-03 16:27:34 +00:00
2020-05-22 11:48:13 +00:00
def clearQueueItems(baseDir: str, queue: []) -> None:
2020-05-22 12:57:15 +00:00
"""Clears the queue for each account
2020-05-22 11:48:13 +00:00
"""
ctr = 0
queue.clear()
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
for account in dirs:
queueDir = baseDir + '/accounts/' + account + '/queue'
2020-06-02 09:05:55 +00:00
if not os.path.isdir(queueDir):
continue
for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
for qfile in queuefiles:
try:
os.remove(os.path.join(queueDir, qfile))
ctr += 1
except BaseException:
pass
2020-05-22 11:48:13 +00:00
if ctr > 0:
print('Removed ' + str(ctr) + ' inbox queue items')
2020-04-03 16:27:34 +00:00
def restoreQueueItems(baseDir: str, queue: []) -> None:
2019-07-12 21:09:23 +00:00
"""Checks the queue for each account and appends filenames
"""
2019-08-15 16:45:07 +00:00
queue.clear()
2020-04-03 16:27:34 +00:00
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
2019-07-12 21:09:23 +00:00
for account in dirs:
2020-04-03 16:27:34 +00:00
queueDir = baseDir + '/accounts/' + account + '/queue'
2020-06-02 09:05:55 +00:00
if not os.path.isdir(queueDir):
continue
for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
for qfile in queuefiles:
queue.append(os.path.join(queueDir, qfile))
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
print('Restored ' + str(len(queue)) + ' inbox queue items')
2019-09-02 21:52:43 +00:00
2020-04-03 16:27:34 +00:00
def runInboxQueueWatchdog(projectVersion: str, httpd) -> None:
2019-09-02 21:52:43 +00:00
"""This tries to keep the inbox thread running even if it dies
"""
print('Starting inbox queue watchdog')
2020-04-03 16:27:34 +00:00
inboxQueueOriginal = httpd.thrInboxQueue.clone(runInboxQueue)
2019-09-02 21:52:43 +00:00
httpd.thrInboxQueue.start()
while True:
2020-03-22 21:16:02 +00:00
time.sleep(20)
2020-04-27 09:41:38 +00:00
if not httpd.thrInboxQueue.isAlive() or httpd.restartInboxQueue:
2020-05-02 10:19:24 +00:00
httpd.restartInboxQueueInProgress = True
2019-09-02 21:52:43 +00:00
httpd.thrInboxQueue.kill()
2020-04-03 16:27:34 +00:00
httpd.thrInboxQueue = inboxQueueOriginal.clone(runInboxQueue)
2020-04-27 09:41:38 +00:00
httpd.inboxQueue.clear()
2019-09-02 21:52:43 +00:00
httpd.thrInboxQueue.start()
print('Restarting inbox queue...')
2020-05-02 10:19:24 +00:00
httpd.restartInboxQueueInProgress = False
httpd.restartInboxQueue = False
2019-09-02 21:52:43 +00:00
2020-04-03 16:27:34 +00:00
def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
projectVersion: str,
baseDir: str, httpPrefix: str, sendThreads: [], postLog: [],
cachedWebfingers: {}, personCache: {}, queue: [],
2020-06-03 20:21:44 +00:00
domain: str,
2020-06-09 11:03:59 +00:00
onionDomain: str, i2pDomain: str, port: int, proxyType: str,
2020-04-03 16:27:34 +00:00
federationList: [],
ocapAlways: bool, maxReplies: int,
domainMaxPostsPerDay: int, accountMaxPostsPerDay: int,
allowDeletion: bool, debug: bool, maxMentions: int,
maxEmoji: int, translate: {}, unitTest: bool,
YTReplacementDomain: str,
2020-04-03 16:27:34 +00:00
acceptedCaps=["inbox:write", "objects:read"]) -> None:
"""Processes received items and moves them to the appropriate
directories
2019-07-04 12:23:53 +00:00
"""
2020-04-03 16:27:34 +00:00
currSessionTime = int(time.time())
sessionLastUpdate = currSessionTime
2020-06-24 09:04:58 +00:00
print('Starting new session when starting inbox queue')
2020-06-09 11:03:59 +00:00
session = createSession(proxyType)
2020-04-03 16:27:34 +00:00
inboxHandle = 'inbox@' + domain
2019-07-04 12:23:53 +00:00
if debug:
print('DEBUG: Inbox queue running')
2019-07-12 21:09:23 +00:00
# if queue processing was interrupted (eg server crash)
# then this loads any outstanding items back into the queue
2020-04-03 16:27:34 +00:00
restoreQueueItems(baseDir, queue)
2019-07-15 10:22:19 +00:00
2020-03-25 10:21:25 +00:00
# keep track of numbers of incoming posts per day
2020-04-03 16:27:34 +00:00
quotasLastUpdateDaily = int(time.time())
quotasDaily = {
2019-07-15 10:22:19 +00:00
'domains': {},
'accounts': {}
}
2020-04-03 16:27:34 +00:00
quotasLastUpdatePerMin = int(time.time())
quotasPerMin = {
2020-03-25 10:36:37 +00:00
'domains': {},
'accounts': {}
}
2020-04-03 16:27:34 +00:00
heartBeatCtr = 0
queueRestoreCtr = 0
2019-09-03 08:46:26 +00:00
2019-07-04 12:23:53 +00:00
while True:
2020-04-16 18:25:59 +00:00
time.sleep(1)
2019-09-03 08:46:26 +00:00
# heartbeat to monitor whether the inbox queue is running
2020-04-03 16:27:34 +00:00
heartBeatCtr += 5
if heartBeatCtr >= 10:
2020-04-16 10:14:05 +00:00
print('>>> Heartbeat Q:' + str(len(queue)) + ' ' +
2020-04-03 16:27:34 +00:00
'{:%F %T}'.format(datetime.datetime.now()))
heartBeatCtr = 0
if len(queue) == 0:
2019-09-03 09:11:33 +00:00
# restore any remaining queue items
2020-04-03 16:27:34 +00:00
queueRestoreCtr += 1
if queueRestoreCtr >= 30:
queueRestoreCtr = 0
restoreQueueItems(baseDir, queue)
2020-04-16 09:49:57 +00:00
continue
2020-04-16 10:14:05 +00:00
2020-04-16 09:49:57 +00:00
currTime = int(time.time())
# recreate the session periodically
2020-06-24 09:40:17 +00:00
if not session or currTime - sessionLastUpdate > 21600:
print('Regenerating inbox queue session at 6hr interval')
2020-06-09 11:03:59 +00:00
session = createSession(proxyType)
2020-06-08 20:18:02 +00:00
if not session:
continue
2020-04-16 09:49:57 +00:00
sessionLastUpdate = currTime
# oldest item first
queue.sort()
queueFilename = queue[0]
if not os.path.isfile(queueFilename):
print("Queue: queue item rejected because it has no file: " +
queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2019-07-04 12:23:53 +00:00
2020-04-16 09:49:57 +00:00
print('Loading queue item ' + queueFilename)
2020-03-22 21:16:02 +00:00
2020-04-16 09:49:57 +00:00
# Load the queue json
queueJson = loadJson(queueFilename, 1)
if not queueJson:
print('Queue: runInboxQueue failed to load inbox queue item ' +
queueFilename)
# Assume that the file is probably corrupt/unreadable
if len(queue) > 0:
queue.pop(0)
# delete the queue file
if os.path.isfile(queueFilename):
try:
os.remove(queueFilename)
except BaseException:
pass
continue
2020-03-22 21:16:02 +00:00
2020-04-16 09:49:57 +00:00
# clear the daily quotas for maximum numbers of received posts
if currTime-quotasLastUpdateDaily > 60 * 60 * 24:
quotasDaily = {
'domains': {},
'accounts': {}
}
quotasLastUpdateDaily = currTime
# clear the per minute quotas for maximum numbers of received posts
if currTime-quotasLastUpdatePerMin > 60:
quotasPerMin = {
'domains': {},
'accounts': {}
}
quotasLastUpdatePerMin = currTime
# limit the number of posts which can arrive per domain per day
postDomain = queueJson['postDomain']
if postDomain:
if domainMaxPostsPerDay > 0:
if quotasDaily['domains'].get(postDomain):
if quotasDaily['domains'][postDomain] > \
domainMaxPostsPerDay:
print('Queue: Quota per day - Maximum posts for ' +
postDomain + ' reached (' +
str(domainMaxPostsPerDay) + ')')
if len(queue) > 0:
try:
os.remove(queueFilename)
except BaseException:
pass
queue.pop(0)
continue
quotasDaily['domains'][postDomain] += 1
else:
quotasDaily['domains'][postDomain] = 1
if quotasPerMin['domains'].get(postDomain):
domainMaxPostsPerMin = \
int(domainMaxPostsPerDay / (24 * 60))
if domainMaxPostsPerMin < 5:
domainMaxPostsPerMin = 5
2020-04-16 09:49:57 +00:00
if quotasPerMin['domains'][postDomain] > \
domainMaxPostsPerMin:
print('Queue: Quota per min - Maximum posts for ' +
postDomain + ' reached (' +
str(domainMaxPostsPerMin) + ')')
if len(queue) > 0:
try:
os.remove(queueFilename)
except BaseException:
pass
queue.pop(0)
continue
quotasPerMin['domains'][postDomain] += 1
else:
quotasPerMin['domains'][postDomain] = 1
if accountMaxPostsPerDay > 0:
postHandle = queueJson['postNickname'] + '@' + postDomain
if quotasDaily['accounts'].get(postHandle):
if quotasDaily['accounts'][postHandle] > \
accountMaxPostsPerDay:
print('Queue: Quota account posts per day -' +
' Maximum posts for ' +
postHandle + ' reached (' +
str(accountMaxPostsPerDay) + ')')
if len(queue) > 0:
try:
os.remove(queueFilename)
except BaseException:
pass
queue.pop(0)
continue
quotasDaily['accounts'][postHandle] += 1
else:
quotasDaily['accounts'][postHandle] = 1
if quotasPerMin['accounts'].get(postHandle):
accountMaxPostsPerMin = \
int(accountMaxPostsPerDay / (24 * 60))
if accountMaxPostsPerMin < 5:
accountMaxPostsPerMin = 5
2020-04-16 09:49:57 +00:00
if quotasPerMin['accounts'][postHandle] > \
accountMaxPostsPerMin:
print('Queue: Quota account posts per min -' +
' Maximum posts for ' +
postHandle + ' reached (' +
str(accountMaxPostsPerMin) + ')')
if len(queue) > 0:
try:
os.remove(queueFilename)
except BaseException:
pass
queue.pop(0)
continue
quotasPerMin['accounts'][postHandle] += 1
else:
quotasPerMin['accounts'][postHandle] = 1
2020-03-25 10:47:13 +00:00
2020-04-16 09:49:57 +00:00
if debug:
if accountMaxPostsPerDay > 0 or domainMaxPostsPerDay > 0:
pprint(quotasDaily)
2019-07-15 10:22:19 +00:00
2020-08-23 14:45:58 +00:00
if queueJson.get('actor'):
print('Obtaining public key for actor ' + queueJson['actor'])
2020-03-22 21:16:02 +00:00
2020-04-16 09:49:57 +00:00
# Try a few times to obtain the public key
pubKey = None
keyId = None
for tries in range(8):
2020-04-03 16:27:34 +00:00
keyId = None
2020-04-16 09:49:57 +00:00
signatureParams = \
queueJson['httpHeaders']['signature'].split(',')
for signatureItem in signatureParams:
if signatureItem.startswith('keyId='):
if '"' in signatureItem:
keyId = signatureItem.split('"')[1]
break
if not keyId:
print('Queue: No keyId in signature: ' +
queueJson['httpHeaders']['signature'])
pubKey = None
break
pubKey = \
getPersonPubKey(baseDir, session, keyId,
personCache, debug,
projectVersion, httpPrefix,
domain, onionDomain)
if pubKey:
2019-07-04 17:31:41 +00:00
if debug:
2020-04-16 09:49:57 +00:00
print('DEBUG: public key: ' + str(pubKey))
break
2019-07-04 17:31:41 +00:00
2019-08-15 08:36:49 +00:00
if debug:
2020-04-16 09:49:57 +00:00
print('DEBUG: Retry ' + str(tries+1) +
' obtaining public key for ' + keyId)
2020-04-16 18:25:59 +00:00
time.sleep(1)
2019-07-04 12:23:53 +00:00
2020-04-16 09:49:57 +00:00
if not pubKey:
print('Queue: public key could not be obtained from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
# check the signature
if debug:
print('DEBUG: checking http headers')
pprint(queueJson['httpHeaders'])
if not verifyPostHeaders(httpPrefix,
pubKey,
queueJson['httpHeaders'],
queueJson['path'], False,
queueJson['digest'],
json.dumps(queueJson['post']),
debug):
print('Queue: Header signature check failed')
pprint(queueJson['httpHeaders'])
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2019-07-04 17:31:41 +00:00
2020-04-16 09:49:57 +00:00
if debug:
print('DEBUG: Signature check success')
2020-03-22 21:16:02 +00:00
2020-04-16 09:49:57 +00:00
# set the id to the same as the post filename
# This makes the filename and the id consistent
# if queueJson['post'].get('id'):
# queueJson['post']['id']=queueJson['id']
if receiveUndo(session,
baseDir, httpPrefix, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
debug,
acceptedCaps=["inbox:write", "objects:read"]):
print('Queue: Undo accepted from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2019-07-17 10:34:00 +00:00
2020-04-16 09:49:57 +00:00
if debug:
print('DEBUG: checking for follow requests')
if receiveFollowRequest(session,
baseDir, httpPrefix, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
debug, projectVersion,
acceptedCaps=["inbox:write",
"objects:read"]):
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
print('Queue: Follow activity for ' + keyId +
2020-06-28 19:04:43 +00:00
' removed from queue')
2020-04-16 09:49:57 +00:00
continue
else:
2019-08-15 16:05:28 +00:00
if debug:
2020-04-16 09:49:57 +00:00
print('DEBUG: No follow requests')
if receiveAcceptReject(session,
baseDir, httpPrefix, domain, port,
sendThreads, postLog,
cachedWebfingers, personCache,
queueJson['post'],
federationList, debug):
print('Queue: Accept/Reject received from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2019-07-06 15:17:21 +00:00
2020-08-20 16:51:48 +00:00
if receiveEventPost(recentPostsCache, session,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
queueJson['postNickname'],
debug):
print('Queue: Event activity accepted from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2020-04-16 09:49:57 +00:00
if receiveUpdate(recentPostsCache, session,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
queueJson['postNickname'],
debug):
print('Queue: Update accepted from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
# get recipients list
recipientsDict, recipientsDictFollowers = \
inboxPostRecipients(baseDir, queueJson['post'],
httpPrefix, domain, port, debug)
if len(recipientsDict.items()) == 0 and \
len(recipientsDictFollowers.items()) == 0:
print('Queue: no recipients were resolved ' +
'for post arriving in inbox')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
2019-07-09 14:20:23 +00:00
2020-04-16 09:49:57 +00:00
# if there are only a small number of followers then
# process them as if they were specifically
# addresses to particular accounts
noOfFollowItems = len(recipientsDictFollowers.items())
if noOfFollowItems > 0:
# always deliver to individual inboxes
if noOfFollowItems < 999999:
if debug:
print('DEBUG: moving ' + str(noOfFollowItems) +
' inbox posts addressed to followers')
for handle, postItem in recipientsDictFollowers.items():
recipientsDict[handle] = postItem
recipientsDictFollowers = {}
# recipientsList = [recipientsDict, recipientsDictFollowers]
if debug:
print('*************************************')
print('Resolved recipients list:')
pprint(recipientsDict)
print('Resolved followers list:')
pprint(recipientsDictFollowers)
print('*************************************')
if queueJson['post'].get('capability'):
if not isinstance(queueJson['post']['capability'], list):
print('Queue: capability on post should be a list')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
2020-04-03 16:27:34 +00:00
if len(queue) > 0:
queue.pop(0)
2019-07-11 12:29:31 +00:00
continue
2020-04-16 09:49:57 +00:00
# Copy any posts addressed to followers into the shared inbox
# this avoid copying file multiple times to potentially many
# individual inboxes
# This obviously bypasses object capabilities and so
# any checking will needs to be handled at the time when inbox
# GET happens on individual accounts.
# See posts.py/createBoxBase
if len(recipientsDictFollowers) > 0:
sharedInboxPostFilename = \
queueJson['destination'].replace(inboxHandle, inboxHandle)
if not os.path.isfile(sharedInboxPostFilename):
saveJson(queueJson['post'], sharedInboxPostFilename)
# for posts addressed to specific accounts
for handle, capsId in recipientsDict.items():
destination = \
queueJson['destination'].replace(inboxHandle, handle)
# check that capabilities are accepted
2019-07-08 23:05:48 +00:00
if queueJson['post'].get('capability'):
2020-04-16 09:49:57 +00:00
capabilityIdList = queueJson['post']['capability']
# does the capability id list within the post
# contain the id of the recipient with this handle?
# Here the capability id begins with the handle,
# so this could also be matched separately, but it's
# probably not necessary
if capsId in capabilityIdList:
inboxAfterCapabilities(recentPostsCache,
maxRecentPosts,
session, keyId, handle,
queueJson['post'],
baseDir, httpPrefix,
sendThreads, postLog,
cachedWebfingers,
personCache, queue,
2020-06-03 20:21:44 +00:00
domain,
onionDomain, i2pDomain,
2020-06-09 11:03:59 +00:00
port, proxyType,
2020-04-16 09:49:57 +00:00
federationList, ocapAlways,
debug, acceptedCaps,
queueFilename, destination,
maxReplies, allowDeletion,
maxMentions, maxEmoji,
translate, unitTest,
YTReplacementDomain)
2019-07-08 23:05:48 +00:00
else:
2020-04-16 09:49:57 +00:00
print('Queue: object capabilities check has failed')
2019-07-09 08:44:24 +00:00
if debug:
2019-08-18 09:39:12 +00:00
pprint(queueJson['post'])
2020-04-16 09:49:57 +00:00
else:
if not ocapAlways:
inboxAfterCapabilities(recentPostsCache,
maxRecentPosts,
session, keyId, handle,
queueJson['post'],
baseDir, httpPrefix,
sendThreads, postLog,
cachedWebfingers,
personCache, queue,
2020-06-03 20:21:44 +00:00
domain,
onionDomain, i2pDomain,
2020-06-09 11:03:59 +00:00
port, proxyType,
2020-04-16 09:49:57 +00:00
federationList, ocapAlways,
debug, acceptedCaps,
queueFilename, destination,
maxReplies, allowDeletion,
maxMentions, maxEmoji,
translate, unitTest,
YTReplacementDomain)
2020-04-16 09:49:57 +00:00
if debug:
pprint(queueJson['post'])
print('No capability list within post')
print('ocapAlways: ' + str(ocapAlways))
2020-03-22 21:16:02 +00:00
2020-04-16 09:49:57 +00:00
print('Queue: Queue post accepted')
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)