epicyon/outbox.py

672 lines
28 KiB
Python

__filename__ = "outbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.2.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Timeline"
import os
from shutil import copyfile
from session import create_session
from auth import create_password
from posts import is_image_media
from posts import outbox_message_create_wrap
from posts import save_post_to_box
from posts import send_to_followers_thread
from posts import send_to_named_addresses_thread
from utils import has_object_stringType
from utils import get_base_content_from_post
from utils import has_object_dict
from utils import get_local_network_addresses
from utils import get_full_domain
from utils import remove_id_ending
from utils import get_domain_from_actor
from utils import dangerous_markup
from utils import is_featured_writer
from utils import load_json
from utils import save_json
from utils import acct_dir
from utils import local_actor_url
from utils import has_actor
from blocking import is_blocked_domain
from blocking import outbox_block
from blocking import outbox_undo_block
from blocking import outbox_mute
from blocking import outbox_undo_mute
from media import replace_you_tube
from media import replace_twitter
from media import get_media_path
from media import create_media_dirs
from inbox import inbox_update_index
from announce import outbox_announce
from announce import outbox_undo_announce
from follow import outbox_undo_follow
from follow import follower_approval_active
from skills import outbox_skills
from availability import outbox_availability
from like import outbox_like
from like import outbox_undo_like
from reaction import outbox_reaction
from reaction import outbox_undo_reaction
from bookmarks import outbox_bookmark
from bookmarks import outbox_undo_bookmark
from delete import outbox_delete
from shares import outbox_share_upload
from shares import outbox_undo_share_upload
from webapp_post import individual_post_as_html
def _person_receive_update_outbox(recent_posts_cache: {},
base_dir: str, http_prefix: str,
nickname: str, domain: str, port: int,
message_json: {}, debug: bool) -> None:
""" Receive an actor update from c2s
For example, setting the PGP key from the desktop client
"""
# these attachments are updatable via c2s
updatableAttachments = ('PGP', 'OpenPGP', 'Email')
if not message_json.get('type'):
return
if not isinstance(message_json['type'], str):
if debug:
print('DEBUG: c2s actor update type is not a string')
return
if message_json['type'] != 'Update':
return
if not has_object_stringType(message_json, debug):
return
if not isinstance(message_json['object']['type'], str):
if debug:
print('DEBUG: c2s actor update object type is not a string')
return
if message_json['object']['type'] != 'Person':
if debug:
print('DEBUG: not a c2s actor update')
return
if not message_json.get('to'):
if debug:
print('DEBUG: c2s actor update has no "to" field')
return
if not has_actor(message_json, debug):
return
if not message_json.get('id'):
if debug:
print('DEBUG: c2s actor update has no id field')
return
if not isinstance(message_json['id'], str):
if debug:
print('DEBUG: c2s actor update id is not a string')
return
domain_full = get_full_domain(domain, port)
actor = local_actor_url(http_prefix, nickname, domain_full)
if len(message_json['to']) != 1:
if debug:
print('DEBUG: c2s actor update - to does not contain one actor ' +
str(message_json['to']))
return
if message_json['to'][0] != actor:
if debug:
print('DEBUG: c2s actor update - to does not contain actor ' +
str(message_json['to']) + ' ' + actor)
return
if not message_json['id'].startswith(actor + '#updates/'):
if debug:
print('DEBUG: c2s actor update - unexpected id ' +
message_json['id'])
return
updatedActorJson = message_json['object']
# load actor from file
actorFilename = acct_dir(base_dir, nickname, domain) + '.json'
if not os.path.isfile(actorFilename):
print('actorFilename not found: ' + actorFilename)
return
actor_json = load_json(actorFilename)
if not actor_json:
return
actorChanged = False
# update fields within actor
if 'attachment' in updatedActorJson:
for newPropertyValue in updatedActorJson['attachment']:
if not newPropertyValue.get('name'):
continue
if newPropertyValue['name'] not in updatableAttachments:
continue
if not newPropertyValue.get('type'):
continue
if not newPropertyValue.get('value'):
continue
if newPropertyValue['type'] != 'PropertyValue':
continue
if 'attachment' not in actor_json:
continue
found = False
for attachIdx in range(len(actor_json['attachment'])):
if actor_json['attachment'][attachIdx]['type'] != \
'PropertyValue':
continue
if actor_json['attachment'][attachIdx]['name'] != \
newPropertyValue['name']:
continue
else:
if actor_json['attachment'][attachIdx]['value'] != \
newPropertyValue['value']:
actor_json['attachment'][attachIdx]['value'] = \
newPropertyValue['value']
actorChanged = True
found = True
break
if not found:
actor_json['attachment'].append({
"name": newPropertyValue['name'],
"type": "PropertyValue",
"value": newPropertyValue['value']
})
actorChanged = True
# save actor to file
if actorChanged:
save_json(actor_json, actorFilename)
if debug:
print('actor saved: ' + actorFilename)
if debug:
print('New attachment: ' + str(actor_json['attachment']))
message_json['object'] = actor_json
if debug:
print('DEBUG: actor update via c2s - ' + nickname + '@' + domain)
def post_message_to_outbox(session, translate: {},
message_json: {}, postToNickname: str,
server, base_dir: str, http_prefix: str,
domain: str, domain_full: str,
onion_domain: str, i2p_domain: str, port: int,
recent_posts_cache: {}, followers_threads: [],
federation_list: [], send_threads: [],
postLog: [], cached_webfingers: {},
person_cache: {}, allow_deletion: bool,
proxy_type: str, version: str, debug: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
allow_local_network_access: bool,
city: str, system_language: str,
shared_items_federated_domains: [],
sharedItemFederationTokens: {},
low_bandwidth: bool,
signing_priv_key_pem: str,
peertube_instances: str, theme: str,
max_like_count: int,
max_recent_posts: int, cw_lists: {},
lists_enabled: str,
content_license_url: str) -> bool:
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
"""
if not message_json.get('type'):
if debug:
print('DEBUG: POST to outbox has no "type" parameter')
return False
if not message_json.get('object') and message_json.get('content'):
if message_json['type'] != 'Create':
# https://www.w3.org/TR/activitypub/#object-without-create
if debug:
print('DEBUG: POST to outbox - adding Create wrapper')
message_json = \
outbox_message_create_wrap(http_prefix,
postToNickname,
domain, port,
message_json)
# check that the outgoing post doesn't contain any markup
# which can be used to implement exploits
if has_object_dict(message_json):
contentStr = get_base_content_from_post(message_json, system_language)
if contentStr:
if dangerous_markup(contentStr, allow_local_network_access):
print('POST to outbox contains dangerous markup: ' +
str(message_json))
return False
if message_json['type'] == 'Create':
if not (message_json.get('id') and
message_json.get('type') and
message_json.get('actor') and
message_json.get('object') and
message_json.get('to')):
if not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the id parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the type parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the actor parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the object parameter ' +
str(message_json))
else:
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the "to" parameter ' +
str(message_json))
return False
# actor should be a string
if not isinstance(message_json['actor'], str):
return False
# actor should look like a url
if '://' not in message_json['actor'] or \
'.' not in message_json['actor']:
return False
# sent by an actor on a local network address?
if not allow_local_network_access:
localNetworkPatternList = get_local_network_addresses()
for localNetworkPattern in localNetworkPatternList:
if localNetworkPattern in message_json['actor']:
return False
testDomain, testPort = get_domain_from_actor(message_json['actor'])
testDomain = get_full_domain(testDomain, testPort)
if is_blocked_domain(base_dir, testDomain):
if debug:
print('DEBUG: domain is blocked: ' + message_json['actor'])
return False
# replace youtube, so that google gets less tracking data
replace_you_tube(message_json, yt_replace_domain, system_language)
# replace twitter, so that twitter posts can be shown without
# having a twitter account
replace_twitter(message_json, twitter_replacement_domain,
system_language)
# https://www.w3.org/TR/activitypub/#create-activity-outbox
message_json['object']['attributedTo'] = message_json['actor']
if message_json['object'].get('attachment'):
attachmentIndex = 0
attach = message_json['object']['attachment'][attachmentIndex]
if attach.get('mediaType'):
fileExtension = 'png'
mediaTypeStr = \
attach['mediaType']
extensions = {
"jpeg": "jpg",
"gif": "gif",
"svg": "svg",
"webp": "webp",
"avif": "avif",
"audio/mpeg": "mp3",
"ogg": "ogg",
"mp4": "mp4",
"webm": "webm",
"ogv": "ogv"
}
for matchExt, ext in extensions.items():
if mediaTypeStr.endswith(matchExt):
fileExtension = ext
break
mediaDir = \
base_dir + '/accounts/' + \
postToNickname + '@' + domain
uploadMediaFilename = mediaDir + '/upload.' + fileExtension
if not os.path.isfile(uploadMediaFilename):
del message_json['object']['attachment']
else:
# generate a path for the uploaded image
mPath = get_media_path()
mediaPath = mPath + '/' + \
create_password(16).lower() + '.' + fileExtension
create_media_dirs(base_dir, mPath)
mediaFilename = base_dir + '/' + mediaPath
# move the uploaded image to its new path
os.rename(uploadMediaFilename, mediaFilename)
# change the url of the attachment
attach['url'] = \
http_prefix + '://' + domain_full + '/' + mediaPath
attach['url'] = \
attach['url'].replace('/media/',
'/system/' +
'media_attachments/files/')
permittedOutboxTypes = (
'Create', 'Announce', 'Like', 'EmojiReact', 'Follow', 'Undo',
'Update', 'Add', 'Remove', 'Block', 'Delete', 'Skill', 'Ignore'
)
if message_json['type'] not in permittedOutboxTypes:
if debug:
print('DEBUG: POST to outbox - ' + message_json['type'] +
' is not a permitted activity type')
return False
if message_json.get('id'):
post_id = remove_id_ending(message_json['id'])
if debug:
print('DEBUG: id attribute exists within POST to outbox')
else:
if debug:
print('DEBUG: No id attribute within POST to outbox')
post_id = None
if debug:
print('DEBUG: save_post_to_box')
if message_json['type'] != 'Upgrade':
outboxName = 'outbox'
# if this is a blog post or an event then save to its own box
if message_json['type'] == 'Create':
if has_object_dict(message_json):
if message_json['object'].get('type'):
if message_json['object']['type'] == 'Article':
outboxName = 'tlblogs'
savedFilename = \
save_post_to_box(base_dir,
http_prefix,
post_id,
postToNickname, domain_full,
message_json, outboxName)
if not savedFilename:
print('WARN: post not saved to outbox ' + outboxName)
return False
# save all instance blogs to the news actor
if postToNickname != 'news' and outboxName == 'tlblogs':
if '/' in savedFilename:
if is_featured_writer(base_dir, postToNickname, domain):
savedPostId = savedFilename.split('/')[-1]
blogsDir = \
base_dir + '/accounts/news@' + domain + '/tlblogs'
if not os.path.isdir(blogsDir):
os.mkdir(blogsDir)
copyfile(savedFilename, blogsDir + '/' + savedPostId)
inbox_update_index('tlblogs', base_dir,
'news@' + domain,
savedFilename, debug)
# clear the citations file if it exists
citationsFilename = \
base_dir + '/accounts/' + \
postToNickname + '@' + domain + '/.citations.txt'
if os.path.isfile(citationsFilename):
try:
os.remove(citationsFilename)
except OSError:
print('EX: post_message_to_outbox unable to delete ' +
citationsFilename)
# The following activity types get added to the index files
indexedActivities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce'
)
if message_json['type'] in indexedActivities:
indexes = [outboxName, "inbox"]
selfActor = \
local_actor_url(http_prefix, postToNickname, domain_full)
for boxNameIndex in indexes:
if not boxNameIndex:
continue
# should this also go to the media timeline?
if boxNameIndex == 'inbox':
if is_image_media(session, base_dir, http_prefix,
postToNickname, domain,
message_json,
translate,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem):
inbox_update_index('tlmedia', base_dir,
postToNickname + '@' + domain,
savedFilename, debug)
if boxNameIndex == 'inbox' and outboxName == 'tlblogs':
continue
# avoid duplicates of the message if already going
# back to the inbox of the same account
if selfActor not in message_json['to']:
# show sent post within the inbox,
# as is the typical convention
inbox_update_index(boxNameIndex, base_dir,
postToNickname + '@' + domain,
savedFilename, debug)
# regenerate the html
useCacheOnly = False
pageNumber = 1
showIndividualPostIcons = True
manuallyApproveFollowers = \
follower_approval_active(base_dir,
postToNickname, domain)
individual_post_as_html(signing_priv_key_pem,
False, recent_posts_cache,
max_recent_posts,
translate, pageNumber,
base_dir, session,
cached_webfingers,
person_cache,
postToNickname, domain, port,
message_json, None, True,
allow_deletion,
http_prefix, __version__,
boxNameIndex,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
allow_local_network_access,
theme, system_language,
max_like_count,
boxNameIndex != 'dm',
showIndividualPostIcons,
manuallyApproveFollowers,
False, True, useCacheOnly,
cw_lists, lists_enabled)
if outbox_announce(recent_posts_cache,
base_dir, message_json, debug):
if debug:
print('DEBUG: Updated announcements (shares) collection ' +
'for the post associated with the Announce activity')
if not server.session:
print('DEBUG: creating new session for c2s')
server.session = create_session(proxy_type)
if not server.session:
print('ERROR: Failed to create session for post_message_to_outbox')
return False
if debug:
print('DEBUG: sending c2s post to followers')
# remove inactive threads
inactiveFollowerThreads = []
for th in followers_threads:
if not th.is_alive():
inactiveFollowerThreads.append(th)
for th in inactiveFollowerThreads:
followers_threads.remove(th)
if debug:
print('DEBUG: ' + str(len(followers_threads)) +
' followers threads active')
# retain up to 200 threads
if len(followers_threads) > 200:
# kill the thread if it is still alive
if followers_threads[0].is_alive():
followers_threads[0].kill()
# remove it from the list
followers_threads.pop(0)
# create a thread to send the post to followers
followersThread = \
send_to_followers_thread(server.session,
base_dir,
postToNickname,
domain, onion_domain, i2p_domain,
port, http_prefix,
federation_list,
send_threads,
postLog,
cached_webfingers,
person_cache,
message_json, debug,
version,
shared_items_federated_domains,
sharedItemFederationTokens,
signing_priv_key_pem)
followers_threads.append(followersThread)
if debug:
print('DEBUG: handle any unfollow requests')
outbox_undo_follow(base_dir, message_json, debug)
if debug:
print('DEBUG: handle skills changes requests')
outbox_skills(base_dir, postToNickname, message_json, debug)
if debug:
print('DEBUG: handle availability changes requests')
outbox_availability(base_dir, postToNickname, message_json, debug)
if debug:
print('DEBUG: handle any like requests')
outbox_like(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any undo like requests')
outbox_undo_like(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any emoji reaction requests')
outbox_reaction(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any undo emoji reaction requests')
outbox_undo_reaction(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any undo announce requests')
outbox_undo_announce(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any bookmark requests')
outbox_bookmark(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any undo bookmark requests')
outbox_undo_bookmark(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle delete requests')
outbox_delete(base_dir, http_prefix,
postToNickname, domain,
message_json, debug,
allow_deletion,
recent_posts_cache)
if debug:
print('DEBUG: handle block requests')
outbox_block(base_dir, http_prefix,
postToNickname, domain,
port,
message_json, debug)
if debug:
print('DEBUG: handle undo block requests')
outbox_undo_block(base_dir, http_prefix,
postToNickname, domain,
port, message_json, debug)
if debug:
print('DEBUG: handle mute requests')
outbox_mute(base_dir, http_prefix,
postToNickname, domain,
port,
message_json, debug,
recent_posts_cache)
if debug:
print('DEBUG: handle undo mute requests')
outbox_undo_mute(base_dir, http_prefix,
postToNickname, domain,
port,
message_json, debug,
recent_posts_cache)
if debug:
print('DEBUG: handle share uploads')
outbox_share_upload(base_dir, http_prefix, postToNickname, domain,
port, message_json, debug, city,
system_language, translate, low_bandwidth,
content_license_url)
if debug:
print('DEBUG: handle undo share uploads')
outbox_undo_share_upload(base_dir, http_prefix,
postToNickname, domain,
port, message_json, debug)
if debug:
print('DEBUG: handle actor updates from c2s')
_person_receive_update_outbox(recent_posts_cache,
base_dir, http_prefix,
postToNickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: sending c2s post to named addresses')
if message_json.get('to'):
print('c2s sender: ' +
postToNickname + '@' + domain + ':' + str(port) +
' recipient: ' + str(message_json['to']))
else:
print('c2s sender: ' +
postToNickname + '@' + domain + ':' + str(port))
namedAddressesThread = \
send_to_named_addresses_thread(server.session, base_dir,
postToNickname,
domain, onion_domain, i2p_domain, port,
http_prefix,
federation_list,
send_threads,
postLog,
cached_webfingers,
person_cache,
message_json, debug,
version,
shared_items_federated_domains,
sharedItemFederationTokens,
signing_priv_key_pem)
followers_threads.append(namedAddressesThread)
return True