epicyon/outbox.py

827 lines
35 KiB
Python

__filename__ = "outbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.5.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Timeline"
import os
from shutil import copyfile
from auth import create_password
from posts import is_image_media
from posts import outbox_message_create_wrap
from posts import save_post_to_box
from posts import send_to_followers_thread
from posts import send_to_named_addresses_thread
from utils import get_attributed_to
from utils import contains_invalid_actor_url_chars
from utils import get_attachment_property_value
from utils import get_account_timezone
from utils import has_object_string_type
from utils import get_base_content_from_post
from utils import has_object_dict
from utils import get_local_network_addresses
from utils import get_full_domain
from utils import remove_id_ending
from utils import get_domain_from_actor
from utils import dangerous_markup
from utils import is_featured_writer
from utils import load_json
from utils import save_json
from utils import acct_dir
from utils import local_actor_url
from utils import has_actor
from utils import is_quote_toot
from utils import get_actor_from_post
from blocking import is_blocked_domain
from blocking import outbox_block
from blocking import outbox_undo_block
from blocking import outbox_mute
from blocking import outbox_undo_mute
from media import replace_you_tube
from media import replace_twitter
from media import get_media_path
from media import create_media_dirs
from inbox import store_hash_tags
from inbox import inbox_update_index
from announce import outbox_announce
from announce import outbox_undo_announce
from follow import outbox_undo_follow
from follow import follower_approval_active
from skills import outbox_skills
from availability import outbox_availability
from like import outbox_like
from like import outbox_undo_like
from reaction import outbox_reaction
from reaction import outbox_undo_reaction
from bookmarks import outbox_bookmark
from bookmarks import outbox_undo_bookmark
from delete import outbox_delete
from shares import outbox_share_upload
from shares import outbox_undo_share_upload
from webapp_post import individual_post_as_html
from speaker import update_speaker
from reading import store_book_events
from reading import has_edition_tag
def _person_receive_update_outbox(base_dir: str, http_prefix: str,
nickname: str, domain: str, port: int,
message_json: {}, debug: bool) -> None:
""" Receive an actor update from c2s
For example, setting the PGP key from the desktop client
"""
# these attachments are updatable via c2s
updatable_attachments = ('PGP', 'OpenPGP', 'Email')
if not message_json.get('type'):
return
if not isinstance(message_json['type'], str):
if debug:
print('DEBUG: c2s actor update type is not a string')
return
if message_json['type'] != 'Update':
return
if not has_object_string_type(message_json, debug):
return
if not isinstance(message_json['object']['type'], str):
if debug:
print('DEBUG: c2s actor update object type is not a string')
return
if message_json['object']['type'] != 'Person':
if debug:
print('DEBUG: not a c2s actor update')
return
if not message_json.get('to'):
if debug:
print('DEBUG: c2s actor update has no "to" field')
return
if not has_actor(message_json, debug):
return
if not message_json.get('id'):
if debug:
print('DEBUG: c2s actor update has no id field')
return
if not isinstance(message_json['id'], str):
if debug:
print('DEBUG: c2s actor update id is not a string')
return
domain_full = get_full_domain(domain, port)
actor = local_actor_url(http_prefix, nickname, domain_full)
if not isinstance(message_json['to'], list):
if debug:
print('DEBUG: c2s actor update - to field is not a list ' +
str(message_json['to']))
return
if len(message_json['to']) != 1:
if debug:
print('DEBUG: c2s actor update - to does not contain one actor ' +
str(message_json['to']))
return
if message_json['to'][0] != actor:
if debug:
print('DEBUG: c2s actor update - to does not contain actor ' +
str(message_json['to']) + ' ' + actor)
return
if not message_json['id'].startswith(actor + '#updates/'):
if debug:
print('DEBUG: c2s actor update - unexpected id ' +
message_json['id'])
return
updated_actor_json = message_json['object']
# load actor from file
actor_filename = acct_dir(base_dir, nickname, domain) + '.json'
if not os.path.isfile(actor_filename):
print('actor_filename not found: ' + actor_filename)
return
actor_json = load_json(actor_filename)
if not actor_json:
return
actor_changed = False
# update fields within actor
if 'attachment' in updated_actor_json:
for new_property_value in updated_actor_json['attachment']:
name_value = None
if new_property_value.get('name'):
name_value = new_property_value['name']
elif new_property_value.get('schema:name'):
name_value = new_property_value['schema:name']
if not name_value:
continue
if name_value not in updatable_attachments:
continue
if not new_property_value.get('type'):
continue
prop_value_name, _ = \
get_attachment_property_value(new_property_value)
if not prop_value_name:
continue
if not new_property_value['type'].endswith('PropertyValue'):
continue
if 'attachment' not in actor_json:
continue
found = False
for attach_idx, _ in enumerate(actor_json['attachment']):
attach_type = actor_json['attachment'][attach_idx]['type']
if not attach_type.endswith('PropertyValue'):
continue
attach_name = ''
if actor_json['attachment'][attach_idx].get('name'):
attach_name = \
actor_json['attachment'][attach_idx]['name']
elif actor_json['attachment'][attach_idx].get('schema:name'):
attach_name = \
actor_json['attachment'][attach_idx]['schema:name']
if attach_name != name_value:
continue
if actor_json['attachment'][attach_idx][prop_value_name] != \
new_property_value[prop_value_name]:
actor_json['attachment'][attach_idx][prop_value_name] = \
new_property_value[prop_value_name]
actor_changed = True
found = True
break
if not found:
actor_json['attachment'].append({
"name": name_value,
"type": "PropertyValue",
"value": new_property_value[prop_value_name]
})
actor_changed = True
# save actor to file
if actor_changed:
save_json(actor_json, actor_filename)
if debug:
print('actor saved: ' + actor_filename)
if debug:
print('New attachment: ' + str(actor_json['attachment']))
message_json['object'] = actor_json
if debug:
print('DEBUG: actor update via c2s - ' + nickname + '@' + domain)
def _capitalize_hashtag(content: str, message_json: {},
system_language: str, translate: {},
original_tag: str,
capitalized_tag: str) -> None:
"""If a nowplaying hashtag exists then ensure it is capitalized
"""
if translate.get(original_tag) and \
translate.get(capitalized_tag):
original_tag = translate[original_tag].replace(' ', '_')
capitalized_tag = translate[capitalized_tag].replace(' ', '_')
if '#' + original_tag not in content:
return
content = content.replace('#' + original_tag, '#' + capitalized_tag)
if 'contentMap' in message_json['object']:
if message_json['object']['contentMap'].get(system_language):
message_json['object']['contentMap'][system_language] = content
message_json['object']['contentMap'][system_language] = content
def post_message_to_outbox(session, translate: {},
message_json: {}, post_to_nickname: str,
server, base_dir: str, http_prefix: str,
domain: str, domain_full: str,
onion_domain: str, i2p_domain: str, port: int,
recent_posts_cache: {}, followers_threads: [],
federation_list: [], send_threads: [],
post_log: [], cached_webfingers: {},
person_cache: {}, allow_deletion: bool,
proxy_type: str, version: str, debug: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
allow_local_network_access: bool,
city: str, system_language: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
low_bandwidth: bool,
signing_priv_key_pem: str,
peertube_instances: str, theme: str,
max_like_count: int,
max_recent_posts: int, cw_lists: {},
lists_enabled: str,
content_license_url: str,
dogwhistles: {},
min_images_for_accounts: [],
buy_sites: {},
sites_unavailable: [],
max_recent_books: int,
books_cache: {},
max_cached_readers: int,
auto_cw_cache: {},
block_federated: []) -> bool:
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
"""
if not message_json.get('type'):
if debug:
print('DEBUG: POST to outbox has no "type" parameter')
return False
if not message_json.get('object') and message_json.get('content'):
if message_json['type'] != 'Create':
# https://www.w3.org/TR/activitypub/#object-without-create
if debug:
print('DEBUG: POST to outbox - adding Create wrapper')
message_json = \
outbox_message_create_wrap(http_prefix,
post_to_nickname,
domain, port,
message_json)
bold_reading = False
if server.bold_reading.get(post_to_nickname):
bold_reading = True
# check that the outgoing post doesn't contain any markup
# which can be used to implement exploits
if has_object_dict(message_json):
# if this is a local only post, is it really local?
if 'localOnly' in message_json['object'] and \
message_json['object'].get('to') and \
message_json['object'].get('attributedTo'):
if message_json['object']['localOnly'] is True:
# check that the to addresses are local
if isinstance(message_json['object']['to'], list):
for to_actor in message_json['object']['to']:
to_domain, to_port = get_domain_from_actor(to_actor)
if not to_domain:
continue
to_domain_full = get_full_domain(to_domain, to_port)
if domain_full != to_domain_full:
print("REJECT: local only post isn't local " +
str(message_json))
return False
# check that the sender is local
local_actor = \
get_attributed_to(message_json['object']['attributedTo'])
local_domain, local_port = get_domain_from_actor(local_actor)
if local_domain:
local_domain_full = \
get_full_domain(local_domain, local_port)
if domain_full != local_domain_full:
print("REJECT: local only post isn't local " +
str(message_json))
return False
if is_quote_toot(message_json, ''):
print('REJECT: POST quote toot ' + str(message_json))
return False
content_str = get_base_content_from_post(message_json, system_language)
if content_str:
_capitalize_hashtag(content_str, message_json,
system_language, translate,
'nowplaying', 'NowPlaying')
if dangerous_markup(content_str, allow_local_network_access, []):
print('POST to outbox contains dangerous markup: ' +
str(message_json))
return False
if message_json['type'] == 'Create':
if not (message_json.get('id') and
message_json.get('type') and
message_json.get('actor') and
message_json.get('object') and
message_json.get('to')):
if not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the id parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the type parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the actor parameter ' +
str(message_json))
elif not message_json.get('id'):
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the object parameter ' +
str(message_json))
else:
if debug:
print('DEBUG: POST to outbox - ' +
'Create does not have the "to" parameter ' +
str(message_json))
return False
# actor should be a string
actor_url = get_actor_from_post(message_json)
if not actor_url:
return False
# actor should look like a url
if '://' not in actor_url or \
'.' not in actor_url:
return False
if contains_invalid_actor_url_chars(actor_url):
return False
# sent by an actor on a local network address?
if not allow_local_network_access:
local_network_pattern_list = get_local_network_addresses()
for local_network_pattern in local_network_pattern_list:
if local_network_pattern in actor_url:
return False
test_domain, test_port = get_domain_from_actor(actor_url)
if test_domain:
test_domain = get_full_domain(test_domain, test_port)
if is_blocked_domain(base_dir, test_domain, None, None):
if debug:
print('DEBUG: domain is blocked: ' + actor_url)
return False
# replace youtube, so that google gets less tracking data
replace_you_tube(message_json, yt_replace_domain, system_language)
# replace twitter, so that twitter posts can be shown without
# having a twitter account
replace_twitter(message_json, twitter_replacement_domain,
system_language)
# https://www.w3.org/TR/activitypub/#create-activity-outbox
message_json['object']['attributedTo'] = actor_url
if message_json['object'].get('attachment'):
attachment_index = 0
attach = message_json['object']['attachment'][attachment_index]
if attach.get('mediaType'):
file_extension = 'png'
media_type_str = \
attach['mediaType']
extensions = {
"jpeg": "jpg",
"jxl": "jxl",
"gif": "gif",
"svg": "svg",
"webp": "webp",
"avif": "avif",
"heic": "heic",
"audio/mpeg": "mp3",
"ogg": "ogg",
"audio/wav": "wav",
"audio/x-wav": "wav",
"audio/x-pn-wave": "wav",
"audio/vnd.wave": "wav",
"flac": "flac",
"opus": "opus",
"audio/speex": "spx",
"audio/x-speex": "spx",
"mp4": "mp4",
"webm": "webm",
"ogv": "ogv"
}
for match_ext, ext in extensions.items():
if media_type_str.endswith(match_ext):
file_extension = ext
break
media_dir = \
base_dir + '/accounts/' + \
post_to_nickname + '@' + domain
upload_media_filename = media_dir + '/upload.' + file_extension
if not os.path.isfile(upload_media_filename):
del message_json['object']['attachment']
else:
# generate a path for the uploaded image
mpath = get_media_path()
media_path = mpath + '/' + \
create_password(16).lower() + '.' + file_extension
create_media_dirs(base_dir, mpath)
media_filename = base_dir + '/' + media_path
# move the uploaded image to its new path
os.rename(upload_media_filename, media_filename)
# change the url of the attachment
attach['url'] = \
http_prefix + '://' + domain_full + '/' + media_path
attach['url'] = \
attach['url'].replace('/media/',
'/system/' +
'media_attachments/files/')
permitted_outbox_types = (
'Create', 'Announce', 'Like', 'EmojiReact', 'Follow', 'Undo',
'Update', 'Add', 'Remove', 'Block', 'Delete', 'Skill', 'Ignore',
'Move', 'Edition'
)
if message_json['type'] not in permitted_outbox_types:
if debug:
print('DEBUG: POST to outbox - ' + message_json['type'] +
' is not a permitted activity type')
return False
if message_json.get('id'):
post_id = remove_id_ending(message_json['id'])
if debug:
print('DEBUG: id attribute exists within POST to outbox')
else:
if debug:
print('DEBUG: No id attribute within POST to outbox')
post_id = None
if debug:
print('DEBUG: save_post_to_box')
is_edited_post = False
if message_json['type'] == 'Update' and \
message_json['object']['type'] in ('Note', 'Event'):
is_edited_post = True
message_json['type'] = 'Create'
outbox_name = 'outbox'
store_hash_tags(base_dir, post_to_nickname, domain,
http_prefix, domain_full,
message_json, translate)
# if this is a blog post or an event then save to its own box
if message_json['type'] == 'Create':
if has_object_dict(message_json):
if message_json['object'].get('type'):
if message_json['object']['type'] == 'Article':
outbox_name = 'tlblogs'
saved_filename = \
save_post_to_box(base_dir,
http_prefix,
post_id,
post_to_nickname, domain_full,
message_json, outbox_name)
if not saved_filename:
print('WARN: post not saved to outbox ' + outbox_name)
return False
actor_url = get_actor_from_post(message_json)
update_speaker(base_dir, http_prefix,
post_to_nickname, domain, domain_full,
message_json, person_cache,
translate, actor_url,
theme, system_language,
outbox_name)
if has_edition_tag(message_json):
store_book_events(base_dir,
message_json,
system_language, [],
translate, debug,
max_recent_books,
books_cache,
max_cached_readers)
# save all instance blogs to the news actor
if post_to_nickname != 'news' and outbox_name == 'tlblogs':
if '/' in saved_filename:
if is_featured_writer(base_dir, post_to_nickname, domain):
saved_post_id = saved_filename.split('/')[-1]
blogs_dir = \
base_dir + '/accounts/news@' + domain + '/tlblogs'
if not os.path.isdir(blogs_dir):
os.mkdir(blogs_dir)
copyfile(saved_filename, blogs_dir + '/' + saved_post_id)
inbox_update_index('tlblogs', base_dir,
'news@' + domain,
saved_filename, debug)
# clear the citations file if it exists
citations_filename = \
base_dir + '/accounts/' + \
post_to_nickname + '@' + domain + '/.citations.txt'
if os.path.isfile(citations_filename):
try:
os.remove(citations_filename)
except OSError:
print('EX: post_message_to_outbox unable to delete ' +
citations_filename)
# The following activity types get added to the index files
indexed_activities = (
'Create', 'Question', 'Note', 'Event', 'EncryptedMessage', 'Article',
'Patch', 'Announce', 'ChatMessage'
)
if message_json['type'] in indexed_activities:
indexes = [outbox_name, "inbox"]
self_actor = \
local_actor_url(http_prefix, post_to_nickname, domain_full)
for box_name_index in indexes:
if not box_name_index:
continue
# should this also go to the media timeline?
if box_name_index == 'inbox':
show_vote_posts = True
show_vote_file = \
acct_dir(base_dir, post_to_nickname, domain) + '/.noVotes'
if os.path.isfile(show_vote_file):
show_vote_posts = False
languages_understood = []
if is_image_media(session, base_dir, http_prefix,
post_to_nickname, domain,
message_json,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
bold_reading,
show_vote_posts,
languages_understood):
inbox_update_index('tlmedia', base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
if box_name_index == 'inbox' and outbox_name == 'tlblogs':
continue
# avoid duplicates of the message if already going
# back to the inbox of the same account
if self_actor not in message_json['to']:
# show sent post within the inbox,
# as is the typical convention
inbox_update_index(box_name_index, base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
# regenerate the html
use_cache_only = False
page_number = 1
show_individual_post_icons = True
manually_approve_followers = \
follower_approval_active(base_dir,
post_to_nickname, domain)
timezone = \
get_account_timezone(base_dir,
post_to_nickname, domain)
mitm = False
if os.path.isfile(saved_filename.replace('.json', '') +
'.mitm'):
mitm = True
minimize_all_images = False
if post_to_nickname in min_images_for_accounts:
minimize_all_images = True
individual_post_as_html(signing_priv_key_pem,
False, recent_posts_cache,
max_recent_posts,
translate, page_number,
base_dir, session,
cached_webfingers,
person_cache,
post_to_nickname, domain, port,
message_json, None, True,
allow_deletion,
http_prefix, __version__,
box_name_index,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
allow_local_network_access,
theme, system_language,
max_like_count,
box_name_index != 'dm',
show_individual_post_icons,
manually_approve_followers,
False, True, use_cache_only,
cw_lists, lists_enabled,
timezone, mitm,
bold_reading, dogwhistles,
minimize_all_images, None,
buy_sites, auto_cw_cache)
if is_edited_post:
message_json['type'] = 'Update'
if outbox_announce(recent_posts_cache,
base_dir, message_json, debug):
if debug:
print('DEBUG: Updated announcements (shares) collection ' +
'for the post associated with the Announce activity')
if debug:
print('DEBUG: sending c2s post to followers')
# remove inactive threads
inactive_follower_threads = []
for thr in followers_threads:
if not thr.is_alive():
inactive_follower_threads.append(thr)
for thr in inactive_follower_threads:
followers_threads.remove(thr)
if debug:
print('DEBUG: ' + str(len(followers_threads)) +
' followers threads active')
# retain up to 200 threads
if len(followers_threads) > 200:
# kill the thread if it is still alive
if followers_threads[0].is_alive():
followers_threads[0].kill()
# remove it from the list
followers_threads.pop(0)
# create a thread to send the post to followers
followers_thread = \
send_to_followers_thread(server, server.session,
server.session_onion,
server.session_i2p,
base_dir,
post_to_nickname,
domain, onion_domain, i2p_domain,
port, http_prefix,
federation_list,
send_threads,
post_log,
cached_webfingers,
person_cache,
message_json, debug,
version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem,
sites_unavailable,
system_language)
followers_threads.append(followers_thread)
if debug:
print('DEBUG: handle any unfollow requests')
outbox_undo_follow(base_dir, message_json, debug)
if debug:
print('DEBUG: handle skills changes requests')
outbox_skills(base_dir, post_to_nickname, message_json, debug)
if debug:
print('DEBUG: handle availability changes requests')
outbox_availability(base_dir, post_to_nickname, message_json, debug)
if debug:
print('DEBUG: handle any like requests')
outbox_like(recent_posts_cache,
base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle any undo like requests')
outbox_undo_like(recent_posts_cache,
base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle any emoji reaction requests')
outbox_reaction(recent_posts_cache,
base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle any undo emoji reaction requests')
outbox_undo_reaction(recent_posts_cache,
base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle any undo announce requests')
outbox_undo_announce(recent_posts_cache,
base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle any bookmark requests')
outbox_bookmark(recent_posts_cache,
base_dir, http_prefix,
post_to_nickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle any undo bookmark requests')
outbox_undo_bookmark(recent_posts_cache,
base_dir, http_prefix,
post_to_nickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: handle delete requests')
outbox_delete(base_dir, http_prefix,
post_to_nickname, domain,
message_json, debug,
allow_deletion,
recent_posts_cache)
if debug:
print('DEBUG: handle block requests')
outbox_block(base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle undo block requests')
outbox_undo_block(base_dir, post_to_nickname, domain, message_json, debug)
if debug:
print('DEBUG: handle mute requests')
outbox_mute(base_dir, http_prefix,
post_to_nickname, domain,
port,
message_json, debug,
recent_posts_cache)
if debug:
print('DEBUG: handle undo mute requests')
outbox_undo_mute(base_dir, http_prefix,
post_to_nickname, domain,
port,
message_json, debug,
recent_posts_cache)
if debug:
print('DEBUG: handle share uploads')
outbox_share_upload(base_dir, http_prefix, post_to_nickname, domain,
port, message_json, debug, city,
system_language, translate, low_bandwidth,
content_license_url, block_federated)
if debug:
print('DEBUG: handle undo share uploads')
outbox_undo_share_upload(base_dir, post_to_nickname, domain,
message_json, debug)
if debug:
print('DEBUG: handle actor updates from c2s')
_person_receive_update_outbox(base_dir, http_prefix,
post_to_nickname, domain, port,
message_json, debug)
if debug:
print('DEBUG: sending c2s post to named addresses')
if message_json.get('to'):
print('c2s sender: ' +
post_to_nickname + '@' + domain + ':' + str(port) +
' recipient: ' + str(message_json['to']))
else:
print('c2s sender: ' +
post_to_nickname + '@' + domain + ':' + str(port))
named_addresses_thread = \
send_to_named_addresses_thread(server, server.session,
server.session_onion,
server.session_i2p,
base_dir, post_to_nickname,
domain, onion_domain, i2p_domain, port,
http_prefix,
federation_list,
send_threads,
post_log,
cached_webfingers,
person_cache,
message_json, debug,
version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem,
proxy_type,
server.followers_sync_cache,
server.sites_unavailable,
server.system_language)
followers_threads.append(named_addresses_thread)
return True