epicyon/outbox.py

574 lines
23 KiB
Python
Raw Normal View History

2020-04-03 17:15:33 +00:00
__filename__ = "outbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
2021-01-26 10:07:42 +00:00
__version__ = "1.2.0"
2020-04-03 17:15:33 +00:00
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
2021-06-15 15:08:12 +00:00
__module_group__ = "Timeline"
2020-01-13 10:35:17 +00:00
import os
from shutil import copyfile
2020-01-13 10:35:17 +00:00
from session import createSession
2020-02-04 20:25:00 +00:00
from auth import createPassword
from posts import isImageMedia
2020-01-13 10:35:17 +00:00
from posts import outboxMessageCreateWrap
from posts import savePostToBox
from posts import sendToFollowersThread
from posts import sendToNamedAddresses
from utils import hasObjectDict
2021-02-15 10:06:49 +00:00
from utils import getLocalNetworkAddresses
2020-12-16 11:04:46 +00:00
from utils import getFullDomain
2020-08-23 11:13:35 +00:00
from utils import removeIdEnding
2020-01-13 10:35:17 +00:00
from utils import getDomainFromActor
from utils import dangerousMarkup
2021-02-13 11:37:02 +00:00
from utils import isFeaturedWriter
2021-03-17 20:18:00 +00:00
from utils import loadJson
from utils import saveJson
2020-01-13 10:35:17 +00:00
from blocking import isBlockedDomain
from blocking import outboxBlock
from blocking import outboxUndoBlock
2021-03-20 21:20:41 +00:00
from blocking import outboxMute
from blocking import outboxUndoMute
2020-01-15 11:06:40 +00:00
from media import replaceYouTube
2020-01-13 10:35:17 +00:00
from media import getMediaPath
from media import createMediaDirs
from inbox import inboxUpdateIndex
from announce import outboxAnnounce
from announce import outboxUndoAnnounce
2020-01-13 10:35:17 +00:00
from follow import outboxUndoFollow
from skills import outboxSkills
from availability import outboxAvailability
from like import outboxLike
from like import outboxUndoLike
from bookmarks import outboxBookmark
from bookmarks import outboxUndoBookmark
from delete import outboxDelete
from shares import outboxShareUpload
from shares import outboxUndoShareUpload
2020-04-03 17:15:33 +00:00
2021-03-17 20:18:00 +00:00
def _outboxPersonReceiveUpdate(recentPostsCache: {},
baseDir: str, httpPrefix: str,
nickname: str, domain: str, port: int,
messageJson: {}, debug: bool) -> None:
""" Receive an actor update from c2s
For example, setting the PGP key from the desktop client
"""
# these attachments are updatable via c2s
updatableAttachments = ('PGP', 'OpenPGP', 'Email')
if not messageJson.get('type'):
return
print("messageJson['type'] " + messageJson['type'])
if messageJson['type'] != 'Update':
return
if not hasObjectDict(messageJson):
2021-03-17 20:18:00 +00:00
if debug:
print('DEBUG: c2s actor update object is not dict')
return
if not messageJson['object'].get('type'):
if debug:
print('DEBUG: c2s actor update - no type')
return
if messageJson['object']['type'] != 'Person':
if debug:
print('DEBUG: not a c2s actor update')
return
if not messageJson.get('to'):
if debug:
print('DEBUG: c2s actor update has no "to" field')
return
if not messageJson.get('actor'):
if debug:
print('DEBUG: c2s actor update has no actor field')
return
if not messageJson.get('id'):
if debug:
print('DEBUG: c2s actor update has no id field')
return
actor = \
httpPrefix + '://' + getFullDomain(domain, port) + '/users/' + nickname
if len(messageJson['to']) != 1:
if debug:
print('DEBUG: c2s actor update - to does not contain one actor ' +
messageJson['to'])
return
if messageJson['to'][0] != actor:
if debug:
print('DEBUG: c2s actor update - to does not contain actor ' +
messageJson['to'] + ' ' + actor)
return
if not messageJson['id'].startswith(actor + '#updates/'):
if debug:
print('DEBUG: c2s actor update - unexpected id ' +
messageJson['id'])
return
updatedActorJson = messageJson['object']
# load actor from file
actorFilename = baseDir + '/accounts/' + nickname + '@' + domain + '.json'
if not os.path.isfile(actorFilename):
print('actorFilename not found: ' + actorFilename)
return
actorJson = loadJson(actorFilename)
if not actorJson:
return
actorChanged = False
# update fields within actor
if 'attachment' in updatedActorJson:
for newPropertyValue in updatedActorJson['attachment']:
if not newPropertyValue.get('name'):
continue
if newPropertyValue['name'] not in updatableAttachments:
continue
if not newPropertyValue.get('type'):
continue
if not newPropertyValue.get('value'):
continue
if newPropertyValue['type'] != 'PropertyValue':
continue
if 'attachment' in actorJson:
found = False
2021-03-17 22:50:17 +00:00
for attachIdx in range(len(actorJson['attachment'])):
if actorJson['attachment'][attachIdx]['type'] != \
'PropertyValue':
2021-03-17 20:18:00 +00:00
continue
2021-03-17 22:50:17 +00:00
if actorJson['attachment'][attachIdx]['name'] != \
newPropertyValue['name']:
2021-03-17 22:36:46 +00:00
continue
else:
2021-03-17 22:50:17 +00:00
if actorJson['attachment'][attachIdx]['value'] != \
newPropertyValue['value']:
actorJson['attachment'][attachIdx]['value'] = \
2021-03-17 22:36:46 +00:00
newPropertyValue['value']
actorChanged = True
2021-03-17 21:46:52 +00:00
found = True
2021-03-17 20:18:00 +00:00
break
if not found:
actorJson['attachment'].append({
"name": newPropertyValue['name'],
"type": "PropertyValue",
"value": newPropertyValue['value']
})
2021-03-17 22:36:46 +00:00
actorChanged = True
2021-03-17 20:18:00 +00:00
# save actor to file
if actorChanged:
saveJson(actorJson, actorFilename)
if debug:
print('actor saved: ' + actorFilename)
if debug:
print('New attachment: ' + str(actorJson['attachment']))
messageJson['object'] = actorJson
if debug:
print('DEBUG: actor update via c2s - ' + nickname + '@' + domain)
def postMessageToOutbox(session, translate: {},
messageJson: {}, postToNickname: str,
2020-04-03 17:15:33 +00:00
server, baseDir: str, httpPrefix: str,
2020-06-03 20:21:44 +00:00
domain: str, domainFull: str,
2020-09-28 10:54:41 +00:00
onionDomain: str, i2pDomain: str, port: int,
2020-04-03 17:15:33 +00:00
recentPostsCache: {}, followersThreads: [],
federationList: [], sendThreads: [],
postLog: [], cachedWebfingers: {},
personCache: {}, allowDeletion: bool,
proxyType: str, version: str, debug: bool,
YTReplacementDomain: str,
showPublishedDateOnly: bool,
2021-05-09 19:11:05 +00:00
allowLocalNetworkAccess: bool,
city: str) -> bool:
2020-01-13 10:35:17 +00:00
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
"""
if not messageJson.get('type'):
if debug:
print('DEBUG: POST to outbox has no "type" parameter')
return False
if not messageJson.get('object') and messageJson.get('content'):
2020-04-03 17:15:33 +00:00
if messageJson['type'] != 'Create':
2020-01-13 10:35:17 +00:00
# https://www.w3.org/TR/activitypub/#object-without-create
if debug:
print('DEBUG: POST to outbox - adding Create wrapper')
2020-04-03 17:15:33 +00:00
messageJson = \
outboxMessageCreateWrap(httpPrefix,
postToNickname,
domain, port,
2020-01-13 10:35:17 +00:00
messageJson)
# check that the outgoing post doesn't contain any markup
# which can be used to implement exploits
if hasObjectDict(messageJson):
if messageJson['object'].get('content'):
if dangerousMarkup(messageJson['object']['content'],
allowLocalNetworkAccess):
print('POST to outbox contains dangerous markup: ' +
str(messageJson))
return False
2020-04-03 17:15:33 +00:00
if messageJson['type'] == 'Create':
if not (messageJson.get('id') and
messageJson.get('type') and
messageJson.get('actor') and
messageJson.get('object') and
2020-01-13 10:35:17 +00:00
messageJson.get('to')):
2020-01-13 12:45:27 +00:00
if not messageJson.get('id'):
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' +
'Create does not have the id parameter ' +
str(messageJson))
2020-01-13 12:45:27 +00:00
elif not messageJson.get('id'):
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' +
'Create does not have the type parameter ' +
str(messageJson))
2020-01-13 12:45:27 +00:00
elif not messageJson.get('id'):
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' +
'Create does not have the actor parameter ' +
str(messageJson))
2020-01-13 12:45:27 +00:00
elif not messageJson.get('id'):
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' +
'Create does not have the object parameter ' +
str(messageJson))
2020-01-13 12:45:27 +00:00
else:
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' +
'Create does not have the "to" parameter ' +
str(messageJson))
2020-01-13 10:35:17 +00:00
return False
2021-02-15 10:06:49 +00:00
# actor should be a string
if not isinstance(messageJson['actor'], str):
return False
# actor should look like a url
if '://' not in messageJson['actor'] or \
'.' not in messageJson['actor']:
return False
# sent by an actor on a local network address?
if not allowLocalNetworkAccess:
localNetworkPatternList = getLocalNetworkAddresses()
for localNetworkPattern in localNetworkPatternList:
if localNetworkPattern in messageJson['actor']:
return False
2020-04-03 17:15:33 +00:00
testDomain, testPort = getDomainFromActor(messageJson['actor'])
2020-12-16 11:04:46 +00:00
testDomain = getFullDomain(testDomain, testPort)
2020-04-03 17:15:33 +00:00
if isBlockedDomain(baseDir, testDomain):
2020-01-13 10:35:17 +00:00
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: domain is blocked: ' + messageJson['actor'])
2020-01-13 10:35:17 +00:00
return False
2020-01-15 11:06:40 +00:00
# replace youtube, so that google gets less tracking data
replaceYouTube(messageJson, YTReplacementDomain)
2020-01-13 10:35:17 +00:00
# https://www.w3.org/TR/activitypub/#create-activity-outbox
2020-04-03 17:15:33 +00:00
messageJson['object']['attributedTo'] = messageJson['actor']
2020-01-13 10:35:17 +00:00
if messageJson['object'].get('attachment'):
2020-04-03 17:15:33 +00:00
attachmentIndex = 0
attach = messageJson['object']['attachment'][attachmentIndex]
if attach.get('mediaType'):
fileExtension = 'png'
mediaTypeStr = \
attach['mediaType']
2020-11-28 20:52:13 +00:00
extensions = {
"jpeg": "jpg",
"gif": "gif",
2021-01-11 22:27:57 +00:00
"svg": "svg",
2020-11-28 20:52:13 +00:00
"webp": "webp",
"avif": "avif",
"audio/mpeg": "mp3",
"ogg": "ogg",
"mp4": "mp4",
"webm": "webm",
"ogv": "ogv"
}
for matchExt, ext in extensions.items():
if mediaTypeStr.endswith(matchExt):
fileExtension = ext
break
2020-04-03 17:15:33 +00:00
mediaDir = \
baseDir + '/accounts/' + \
postToNickname + '@' + domain
uploadMediaFilename = mediaDir + '/upload.' + fileExtension
2020-01-13 10:35:17 +00:00
if not os.path.isfile(uploadMediaFilename):
del messageJson['object']['attachment']
else:
# generate a path for the uploaded image
2020-04-03 17:15:33 +00:00
mPath = getMediaPath()
mediaPath = mPath + '/' + \
createPassword(32) + '.' + fileExtension
createMediaDirs(baseDir, mPath)
mediaFilename = baseDir + '/' + mediaPath
2020-01-13 10:35:17 +00:00
# move the uploaded image to its new path
2020-04-03 17:15:33 +00:00
os.rename(uploadMediaFilename, mediaFilename)
2020-01-13 10:35:17 +00:00
# change the url of the attachment
2020-04-03 17:15:33 +00:00
attach['url'] = \
httpPrefix + '://' + domainFull + '/' + mediaPath
2020-01-13 10:35:17 +00:00
2020-04-03 17:15:33 +00:00
permittedOutboxTypes = ('Create', 'Announce', 'Like', 'Follow', 'Undo',
'Update', 'Add', 'Remove', 'Block', 'Delete',
2021-07-04 11:02:08 +00:00
'Skill', 'Ignore')
2020-01-13 10:35:17 +00:00
if messageJson['type'] not in permittedOutboxTypes:
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: POST to outbox - ' + messageJson['type'] +
2020-01-13 10:35:17 +00:00
' is not a permitted activity type')
return False
if messageJson.get('id'):
2020-08-23 11:13:35 +00:00
postId = removeIdEnding(messageJson['id'])
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: id attribute exists within POST to outbox')
else:
if debug:
print('DEBUG: No id attribute within POST to outbox')
2020-04-03 17:15:33 +00:00
postId = None
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: savePostToBox')
2020-04-03 17:15:33 +00:00
if messageJson['type'] != 'Upgrade':
outboxName = 'outbox'
2020-02-24 22:50:55 +00:00
2020-08-23 11:13:35 +00:00
# if this is a blog post or an event then save to its own box
2020-04-03 17:15:33 +00:00
if messageJson['type'] == 'Create':
if hasObjectDict(messageJson):
if messageJson['object'].get('type'):
if messageJson['object']['type'] == 'Article':
outboxName = 'tlblogs'
2020-04-03 17:15:33 +00:00
savedFilename = \
savePostToBox(baseDir,
httpPrefix,
postId,
postToNickname, domainFull,
messageJson, outboxName)
2020-08-25 20:20:56 +00:00
if not savedFilename:
print('WARN: post not saved to outbox ' + outboxName)
return False
# save all instance blogs to the news actor
if postToNickname != 'news' and outboxName == 'tlblogs':
2020-11-28 12:13:04 +00:00
if '/' in savedFilename:
2021-02-13 11:37:02 +00:00
if isFeaturedWriter(baseDir, postToNickname, domain):
savedPostId = savedFilename.split('/')[-1]
blogsDir = \
baseDir + '/accounts/news@' + domain + '/tlblogs'
if not os.path.isdir(blogsDir):
os.mkdir(blogsDir)
copyfile(savedFilename, blogsDir + '/' + savedPostId)
inboxUpdateIndex('tlblogs', baseDir,
'news@' + domain,
savedFilename, debug)
2020-11-28 12:13:04 +00:00
# clear the citations file if it exists
citationsFilename = \
baseDir + '/accounts/' + \
postToNickname + '@' + domain + '/.citations.txt'
if os.path.isfile(citationsFilename):
os.remove(citationsFilename)
2021-07-04 11:11:15 +00:00
# The following activity types get added to the index files
indexedActivities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce'
)
if messageJson['type'] in indexedActivities:
2020-04-03 17:15:33 +00:00
indexes = [outboxName, "inbox"]
2020-08-25 17:28:29 +00:00
selfActor = \
2020-08-25 20:09:14 +00:00
httpPrefix + '://' + domainFull + '/users/' + postToNickname
for boxNameIndex in indexes:
2020-08-25 20:20:56 +00:00
if not boxNameIndex:
continue
# should this also go to the media timeline?
if boxNameIndex == 'inbox':
if isImageMedia(session, baseDir, httpPrefix,
postToNickname, domain,
messageJson,
translate, YTReplacementDomain,
allowLocalNetworkAccess,
2021-03-14 19:32:11 +00:00
recentPostsCache, debug):
inboxUpdateIndex('tlmedia', baseDir,
postToNickname + '@' + domain,
savedFilename, debug)
2020-08-25 14:43:13 +00:00
if boxNameIndex == 'inbox' and outboxName == 'tlblogs':
continue
2021-06-22 20:30:27 +00:00
# avoid duplicates of the message if already going
# back to the inbox of the same account
2021-06-22 20:42:39 +00:00
if selfActor not in messageJson['to']:
# show sent post within the inbox,
# as is the typical convention
inboxUpdateIndex(boxNameIndex, baseDir,
postToNickname + '@' + domain,
savedFilename, debug)
2020-04-03 17:15:33 +00:00
if outboxAnnounce(recentPostsCache,
baseDir, messageJson, debug):
2020-01-13 10:35:17 +00:00
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: Updated announcements (shares) collection ' +
'for the post associated with the Announce activity')
2020-01-13 10:35:17 +00:00
if not server.session:
2020-06-24 09:04:58 +00:00
print('DEBUG: creating new session for c2s')
2020-06-09 11:03:59 +00:00
server.session = createSession(proxyType)
if not server.session:
print('ERROR: Failed to create session for postMessageToOutbox')
return False
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: sending c2s post to followers')
# remove inactive threads
2020-04-03 17:15:33 +00:00
inactiveFollowerThreads = []
2020-01-13 10:35:17 +00:00
for th in followersThreads:
if not th.is_alive():
inactiveFollowerThreads.append(th)
for th in inactiveFollowerThreads:
followersThreads.remove(th)
if debug:
2020-04-03 17:15:33 +00:00
print('DEBUG: ' + str(len(followersThreads)) +
' followers threads active')
2020-01-20 12:43:34 +00:00
# retain up to 200 threads
2020-04-03 17:15:33 +00:00
if len(followersThreads) > 200:
2020-01-13 10:35:17 +00:00
# kill the thread if it is still alive
if followersThreads[0].is_alive():
followersThreads[0].kill()
# remove it from the list
followersThreads.pop(0)
# create a thread to send the post to followers
2020-04-03 17:15:33 +00:00
followersThread = \
sendToFollowersThread(server.session,
baseDir,
postToNickname,
2020-06-03 20:21:44 +00:00
domain, onionDomain, i2pDomain,
2020-04-03 17:15:33 +00:00
port, httpPrefix,
federationList,
sendThreads,
postLog,
cachedWebfingers,
personCache,
messageJson, debug,
2020-01-13 10:35:17 +00:00
version)
followersThreads.append(followersThread)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle any unfollow requests')
2020-04-03 17:15:33 +00:00
outboxUndoFollow(baseDir, messageJson, debug)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle skills changes requests')
2020-04-03 17:15:33 +00:00
outboxSkills(baseDir, postToNickname, messageJson, debug)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle availability changes requests')
2020-04-03 17:15:33 +00:00
outboxAvailability(baseDir, postToNickname, messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle any like requests')
2020-04-03 17:15:33 +00:00
outboxLike(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle any undo like requests')
2020-04-03 17:15:33 +00:00
outboxUndoLike(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
if debug:
print('DEBUG: handle any undo announce requests')
outboxUndoAnnounce(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle any bookmark requests')
2020-04-03 17:15:33 +00:00
outboxBookmark(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle any undo bookmark requests')
2020-04-03 17:15:33 +00:00
outboxUndoBookmark(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
2020-03-22 21:16:02 +00:00
print('DEBUG: handle delete requests')
2020-04-03 17:15:33 +00:00
outboxDelete(baseDir, httpPrefix,
postToNickname, domain,
messageJson, debug,
allowDeletion,
recentPostsCache)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle block requests')
2020-04-03 17:15:33 +00:00
outboxBlock(baseDir, httpPrefix,
postToNickname, domain,
2020-01-13 10:35:17 +00:00
port,
2020-04-03 17:15:33 +00:00
messageJson, debug)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle undo block requests')
2020-04-03 17:15:33 +00:00
outboxUndoBlock(baseDir, httpPrefix,
postToNickname, domain,
port, messageJson, debug)
2020-02-04 20:11:19 +00:00
2021-03-20 21:20:41 +00:00
if debug:
print('DEBUG: handle mute requests')
outboxMute(baseDir, httpPrefix,
postToNickname, domain,
port,
messageJson, debug,
recentPostsCache)
if debug:
print('DEBUG: handle undo mute requests')
outboxUndoMute(baseDir, httpPrefix,
postToNickname, domain,
port,
messageJson, debug,
recentPostsCache)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle share uploads')
2020-04-03 17:15:33 +00:00
outboxShareUpload(baseDir, httpPrefix,
postToNickname, domain,
2021-05-09 19:11:05 +00:00
port, messageJson, debug, city)
2020-02-04 20:11:19 +00:00
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: handle undo share uploads')
2020-04-03 17:15:33 +00:00
outboxUndoShareUpload(baseDir, httpPrefix,
postToNickname, domain,
port, messageJson, debug)
2020-02-04 20:11:19 +00:00
2021-03-17 20:18:00 +00:00
if debug:
print('DEBUG: handle actor updates from c2s')
_outboxPersonReceiveUpdate(recentPostsCache,
baseDir, httpPrefix,
postToNickname, domain, port,
messageJson, debug)
2020-01-13 10:35:17 +00:00
if debug:
print('DEBUG: sending c2s post to named addresses')
2020-02-04 20:11:19 +00:00
if messageJson.get('to'):
2020-04-03 17:15:33 +00:00
print('c2s sender: ' +
postToNickname + '@' + domain + ':' + str(port) +
' recipient: ' + str(messageJson['to']))
2020-02-04 20:11:19 +00:00
else:
2020-04-03 17:15:33 +00:00
print('c2s sender: ' +
postToNickname + '@' + domain + ':' + str(port))
sendToNamedAddresses(server.session, baseDir,
postToNickname,
2020-06-03 20:21:44 +00:00
domain, onionDomain, i2pDomain, port,
2020-04-03 17:15:33 +00:00
httpPrefix,
federationList,
sendThreads,
postLog,
cachedWebfingers,
personCache,
messageJson, debug,
2020-02-21 13:18:14 +00:00
version)
2020-01-13 10:35:17 +00:00
return True