epicyon/outbox.py

__filename__ = "outbox.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.2.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Timeline"

import os
from shutil import copyfile
from session import create_session
from auth import create_password
from posts import is_image_media
from posts import outbox_message_create_wrap
from posts import save_post_to_box
from posts import send_to_followers_thread
from posts import send_to_named_addresses_thread
from utils import has_object_stringType
from utils import get_base_content_from_post
from utils import has_object_dict
from utils import get_local_network_addresses
from utils import get_full_domain
from utils import remove_id_ending
from utils import get_domain_from_actor
from utils import dangerous_markup
from utils import is_featured_writer
from utils import load_json
from utils import save_json
from utils import acct_dir
from utils import local_actor_url
from utils import has_actor
from blocking import is_blocked_domain
from blocking import outbox_block
from blocking import outbox_undo_block
from blocking import outbox_mute
from blocking import outbox_undo_mute
from media import replace_you_tube
from media import replace_twitter
from media import get_media_path
from media import create_media_dirs
from inbox import inbox_update_index
from announce import outbox_announce
from announce import outbox_undo_announce
from follow import outbox_undo_follow
from follow import follower_approval_active
from skills import outbox_skills
from availability import outbox_availability
from like import outbox_like
from like import outbox_undo_like
from reaction import outbox_reaction
from reaction import outbox_undo_reaction
from bookmarks import outbox_bookmark
from bookmarks import outbox_undo_bookmark
from delete import outbox_delete
from shares import outbox_share_upload
from shares import outbox_undo_share_upload
from webapp_post import individual_post_as_html


def _person_receive_update_outbox(recent_posts_cache: {},
                                  base_dir: str, http_prefix: str,
                                  nickname: str, domain: str, port: int,
                                  message_json: {}, debug: bool) -> None:
    """ Receive an actor update from c2s
    For example, setting the PGP key from the desktop client
    """
    # these attachments are updatable via c2s
    updatableAttachments = ('PGP', 'OpenPGP', 'Email')

    if not message_json.get('type'):
        return
    if not isinstance(message_json['type'], str):
        if debug:
            print('DEBUG: c2s actor update type is not a string')
        return
    if message_json['type'] != 'Update':
        return
    if not has_object_stringType(message_json, debug):
        return
    if not isinstance(message_json['object']['type'], str):
        if debug:
            print('DEBUG: c2s actor update object type is not a string')
        return
    if message_json['object']['type'] != 'Person':
        if debug:
            print('DEBUG: not a c2s actor update')
        return
    if not message_json.get('to'):
        if debug:
            print('DEBUG: c2s actor update has no "to" field')
        return
    if not has_actor(message_json, debug):
        return
    if not message_json.get('id'):
        if debug:
            print('DEBUG: c2s actor update has no id field')
        return
    if not isinstance(message_json['id'], str):
        if debug:
            print('DEBUG: c2s actor update id is not a string')
        return
    domain_full = get_full_domain(domain, port)
    actor = local_actor_url(http_prefix, nickname, domain_full)
    if len(message_json['to']) != 1:
        if debug:
            print('DEBUG: c2s actor update - to does not contain one actor ' +
                  str(message_json['to']))
        return
    if message_json['to'][0] != actor:
        if debug:
            print('DEBUG: c2s actor update - to does not contain actor ' +
                  str(message_json['to']) + ' ' + actor)
        return
    if not message_json['id'].startswith(actor + '#updates/'):
        if debug:
            print('DEBUG: c2s actor update - unexpected id ' +
                  message_json['id'])
        return
    updatedActorJson = message_json['object']
    # load actor from file
    actorFilename = acct_dir(base_dir, nickname, domain) + '.json'
    if not os.path.isfile(actorFilename):
        print('actorFilename not found: ' + actorFilename)
        return
    actor_json = load_json(actorFilename)
    if not actor_json:
        return
    actorChanged = False
    # update fields within actor
    if 'attachment' in updatedActorJson:
        for newPropertyValue in updatedActorJson['attachment']:
            if not newPropertyValue.get('name'):
                continue
            if newPropertyValue['name'] not in updatableAttachments:
                continue
            if not newPropertyValue.get('type'):
                continue
            if not newPropertyValue.get('value'):
                continue
            if newPropertyValue['type'] != 'PropertyValue':
                continue
            if 'attachment' not in actor_json:
                continue
            found = False
            for attachIdx in range(len(actor_json['attachment'])):
                if actor_json['attachment'][attachIdx]['type'] != \
                   'PropertyValue':
                    continue
                if actor_json['attachment'][attachIdx]['name'] != \
                   newPropertyValue['name']:
                    continue
                else:
                    if actor_json['attachment'][attachIdx]['value'] != \
                       newPropertyValue['value']:
                        actor_json['attachment'][attachIdx]['value'] = \
                            newPropertyValue['value']
                        actorChanged = True
                    found = True
                    break
            if not found:
                actor_json['attachment'].append({
                    "name": newPropertyValue['name'],
                    "type": "PropertyValue",
                    "value": newPropertyValue['value']
                })
                actorChanged = True
    # save actor to file
    if actorChanged:
        save_json(actor_json, actorFilename)
        if debug:
            print('actor saved: ' + actorFilename)
    if debug:
        print('New attachment: ' + str(actor_json['attachment']))
    message_json['object'] = actor_json
    if debug:
        print('DEBUG: actor update via c2s - ' + nickname + '@' + domain)


def post_message_to_outbox(session, translate: {},
                           message_json: {}, postToNickname: str,
                           server, base_dir: str, http_prefix: str,
                           domain: str, domain_full: str,
                           onion_domain: str, i2p_domain: str, port: int,
                           recent_posts_cache: {}, followers_threads: [],
                           federation_list: [], send_threads: [],
                           postLog: [], cached_webfingers: {},
                           person_cache: {}, allow_deletion: bool,
                           proxy_type: str, version: str, debug: bool,
                           yt_replace_domain: str,
                           twitter_replacement_domain: str,
                           show_published_date_only: bool,
                           allow_local_network_access: bool,
                           city: str, system_language: str,
                           shared_items_federated_domains: [],
                           sharedItemFederationTokens: {},
                           low_bandwidth: bool,
                           signing_priv_key_pem: str,
                           peertube_instances: str, theme: str,
                           max_like_count: int,
                           max_recent_posts: int, cw_lists: {},
                           lists_enabled: str,
                           content_license_url: str) -> bool:
    """post is received by the outbox
    Client to server message post
    https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
    """
    if not message_json.get('type'):
        if debug:
            print('DEBUG: POST to outbox has no "type" parameter')
        return False
    if not message_json.get('object') and message_json.get('content'):
        if message_json['type'] != 'Create':
            # https://www.w3.org/TR/activitypub/#object-without-create
            if debug:
                print('DEBUG: POST to outbox - adding Create wrapper')
            message_json = \
                outbox_message_create_wrap(http_prefix,
                                           postToNickname,
                                           domain, port,
                                           message_json)

    # check that the outgoing post doesn't contain any markup
    # which can be used to implement exploits
    if has_object_dict(message_json):
        contentStr = get_base_content_from_post(message_json, system_language)
        if contentStr:
            if dangerous_markup(contentStr, allow_local_network_access):
                print('POST to outbox contains dangerous markup: ' +
                      str(message_json))
                return False

    if message_json['type'] == 'Create':
        if not (message_json.get('id') and
                message_json.get('type') and
                message_json.get('actor') and
                message_json.get('object') and
                message_json.get('to')):
            if not message_json.get('id'):
                if debug:
                    print('DEBUG: POST to outbox - ' +
                          'Create does not have the id parameter ' +
                          str(message_json))
            elif not message_json.get('id'):
                if debug:
                    print('DEBUG: POST to outbox - ' +
                          'Create does not have the type parameter ' +
                          str(message_json))
            elif not message_json.get('id'):
                if debug:
                    print('DEBUG: POST to outbox - ' +
                          'Create does not have the actor parameter ' +
                          str(message_json))
            elif not message_json.get('id'):
                if debug:
                    print('DEBUG: POST to outbox - ' +
                          'Create does not have the object parameter ' +
                          str(message_json))
            else:
                if debug:
                    print('DEBUG: POST to outbox - ' +
                          'Create does not have the "to" parameter ' +
                          str(message_json))
            return False

        # actor should be a string
        if not isinstance(message_json['actor'], str):
            return False

        # actor should look like a url
        if '://' not in message_json['actor'] or \
           '.' not in message_json['actor']:
            return False

        # sent by an actor on a local network address?
        if not allow_local_network_access:
            localNetworkPatternList = get_local_network_addresses()
            for localNetworkPattern in localNetworkPatternList:
                if localNetworkPattern in message_json['actor']:
                    return False

        testDomain, testPort = get_domain_from_actor(message_json['actor'])
        testDomain = get_full_domain(testDomain, testPort)
        if is_blocked_domain(base_dir, testDomain):
            if debug:
                print('DEBUG: domain is blocked: ' + message_json['actor'])
            return False
        # replace youtube, so that google gets less tracking data
        replace_you_tube(message_json, yt_replace_domain, system_language)
        # replace twitter, so that twitter posts can be shown without
        # having a twitter account
        replace_twitter(message_json, twitter_replacement_domain,
                        system_language)
        # https://www.w3.org/TR/activitypub/#create-activity-outbox
        message_json['object']['attributedTo'] = message_json['actor']
        if message_json['object'].get('attachment'):
            attachmentIndex = 0
            attach = message_json['object']['attachment'][attachmentIndex]
            if attach.get('mediaType'):
                fileExtension = 'png'
                mediaTypeStr = \
                    attach['mediaType']

                extensions = {
                    "jpeg": "jpg",
                    "gif": "gif",
                    "svg": "svg",
                    "webp": "webp",
                    "avif": "avif",
                    "audio/mpeg": "mp3",
                    "ogg": "ogg",
                    "mp4": "mp4",
                    "webm": "webm",
                    "ogv": "ogv"
                }
                for matchExt, ext in extensions.items():
                    if mediaTypeStr.endswith(matchExt):
                        fileExtension = ext
                        break

                mediaDir = \
                    base_dir + '/accounts/' + \
                    postToNickname + '@' + domain
                uploadMediaFilename = mediaDir + '/upload.' + fileExtension
                if not os.path.isfile(uploadMediaFilename):
                    del message_json['object']['attachment']
                else:
                    # generate a path for the uploaded image
                    mPath = get_media_path()
                    mediaPath = mPath + '/' + \
                        create_password(16).lower() + '.' + fileExtension
                    create_media_dirs(base_dir, mPath)
                    mediaFilename = base_dir + '/' + mediaPath
                    # move the uploaded image to its new path
                    os.rename(uploadMediaFilename, mediaFilename)
                    # change the url of the attachment
                    attach['url'] = \
                        http_prefix + '://' + domain_full + '/' + mediaPath
                    attach['url'] = \
                        attach['url'].replace('/media/',
                                              '/system/' +
                                              'media_attachments/files/')

    permittedOutboxTypes = (
        'Create', 'Announce', 'Like', 'EmojiReact', 'Follow', 'Undo',
        'Update', 'Add', 'Remove', 'Block', 'Delete', 'Skill', 'Ignore'
    )
    if message_json['type'] not in permittedOutboxTypes:
        if debug:
            print('DEBUG: POST to outbox - ' + message_json['type'] +
                  ' is not a permitted activity type')
        return False
    if message_json.get('id'):
        post_id = remove_id_ending(message_json['id'])
        if debug:
            print('DEBUG: id attribute exists within POST to outbox')
    else:
        if debug:
            print('DEBUG: No id attribute within POST to outbox')
        post_id = None
    if debug:
        print('DEBUG: save_post_to_box')
    if message_json['type'] != 'Upgrade':
        outboxName = 'outbox'

        # if this is a blog post or an event then save to its own box
        if message_json['type'] == 'Create':
            if has_object_dict(message_json):
                if message_json['object'].get('type'):
                    if message_json['object']['type'] == 'Article':
                        outboxName = 'tlblogs'

        savedFilename = \
            save_post_to_box(base_dir,
                             http_prefix,
                             post_id,
                             postToNickname, domain_full,
                             message_json, outboxName)
        if not savedFilename:
            print('WARN: post not saved to outbox ' + outboxName)
            return False

        # save all instance blogs to the news actor
        if postToNickname != 'news' and outboxName == 'tlblogs':
            if '/' in savedFilename:
                if is_featured_writer(base_dir, postToNickname, domain):
                    savedPostId = savedFilename.split('/')[-1]
                    blogsDir = \
                        base_dir + '/accounts/news@' + domain + '/tlblogs'
                    if not os.path.isdir(blogsDir):
                        os.mkdir(blogsDir)
                    copyfile(savedFilename, blogsDir + '/' + savedPostId)
                    inbox_update_index('tlblogs', base_dir,
                                       'news@' + domain,
                                       savedFilename, debug)

                # clear the citations file if it exists
                citationsFilename = \
                    base_dir + '/accounts/' + \
                    postToNickname + '@' + domain + '/.citations.txt'
                if os.path.isfile(citationsFilename):
                    try:
                        os.remove(citationsFilename)
                    except OSError:
                        print('EX: post_message_to_outbox unable to delete ' +
                              citationsFilename)

        # The following activity types get added to the index files
        indexedActivities = (
            'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
            'Patch', 'Announce'
        )
        if message_json['type'] in indexedActivities:
            indexes = [outboxName, "inbox"]
            selfActor = \
                local_actor_url(http_prefix, postToNickname, domain_full)
            for boxNameIndex in indexes:
                if not boxNameIndex:
                    continue

                # should this also go to the media timeline?
                if boxNameIndex == 'inbox':
                    if is_image_media(session, base_dir, http_prefix,
                                      postToNickname, domain,
                                      message_json,
                                      translate,
                                      yt_replace_domain,
                                      twitter_replacement_domain,
                                      allow_local_network_access,
                                      recent_posts_cache, debug,
                                      system_language,
                                      domain_full, person_cache,
                                      signing_priv_key_pem):
                        inbox_update_index('tlmedia', base_dir,
                                           postToNickname + '@' + domain,
                                           savedFilename, debug)

                if boxNameIndex == 'inbox' and outboxName == 'tlblogs':
                    continue

                # avoid duplicates of the message if already going
                # back to the inbox of the same account
                if selfActor not in message_json['to']:
                    # show sent post within the inbox,
                    # as is the typical convention
                    inbox_update_index(boxNameIndex, base_dir,
                                       postToNickname + '@' + domain,
                                       savedFilename, debug)

                    # regenerate the html
                    useCacheOnly = False
                    pageNumber = 1
                    showIndividualPostIcons = True
                    manuallyApproveFollowers = \
                        follower_approval_active(base_dir,
                                                 postToNickname, domain)
                    individual_post_as_html(signing_priv_key_pem,
                                            False, recent_posts_cache,
                                            max_recent_posts,
                                            translate, pageNumber,
                                            base_dir, session,
                                            cached_webfingers,
                                            person_cache,
                                            postToNickname, domain, port,
                                            message_json, None, True,
                                            allow_deletion,
                                            http_prefix, __version__,
                                            boxNameIndex,
                                            yt_replace_domain,
                                            twitter_replacement_domain,
                                            show_published_date_only,
                                            peertube_instances,
                                            allow_local_network_access,
                                            theme, system_language,
                                            max_like_count,
                                            boxNameIndex != 'dm',
                                            showIndividualPostIcons,
                                            manuallyApproveFollowers,
                                            False, True, useCacheOnly,
                                            cw_lists, lists_enabled)

    if outbox_announce(recent_posts_cache,
                       base_dir, message_json, debug):
        if debug:
            print('DEBUG: Updated announcements (shares) collection ' +
                  'for the post associated with the Announce activity')
    if not server.session:
        print('DEBUG: creating new session for c2s')
        server.session = create_session(proxy_type)
        if not server.session:
            print('ERROR: Failed to create session for post_message_to_outbox')
            return False
    if debug:
        print('DEBUG: sending c2s post to followers')
    # remove inactive threads
    inactiveFollowerThreads = []
    for th in followers_threads:
        if not th.is_alive():
            inactiveFollowerThreads.append(th)
    for th in inactiveFollowerThreads:
        followers_threads.remove(th)
    if debug:
        print('DEBUG: ' + str(len(followers_threads)) +
              ' followers threads active')
    # retain up to 200 threads
    if len(followers_threads) > 200:
        # kill the thread if it is still alive
        if followers_threads[0].is_alive():
            followers_threads[0].kill()
        # remove it from the list
        followers_threads.pop(0)
    # create a thread to send the post to followers
    followersThread = \
        send_to_followers_thread(server.session,
                                 base_dir,
                                 postToNickname,
                                 domain, onion_domain, i2p_domain,
                                 port, http_prefix,
                                 federation_list,
                                 send_threads,
                                 postLog,
                                 cached_webfingers,
                                 person_cache,
                                 message_json, debug,
                                 version,
                                 shared_items_federated_domains,
                                 sharedItemFederationTokens,
                                 signing_priv_key_pem)
    followers_threads.append(followersThread)

    if debug:
        print('DEBUG: handle any unfollow requests')
    outbox_undo_follow(base_dir, message_json, debug)

    if debug:
        print('DEBUG: handle skills changes requests')
    outbox_skills(base_dir, postToNickname, message_json, debug)

    if debug:
        print('DEBUG: handle availability changes requests')
    outbox_availability(base_dir, postToNickname, message_json, debug)

    if debug:
        print('DEBUG: handle any like requests')
    outbox_like(recent_posts_cache,
                base_dir, http_prefix,
                postToNickname, domain, port,
                message_json, debug)
    if debug:
        print('DEBUG: handle any undo like requests')
    outbox_undo_like(recent_posts_cache,
                     base_dir, http_prefix,
                     postToNickname, domain, port,
                     message_json, debug)

    if debug:
        print('DEBUG: handle any emoji reaction requests')
    outbox_reaction(recent_posts_cache,
                    base_dir, http_prefix,
                    postToNickname, domain, port,
                    message_json, debug)
    if debug:
        print('DEBUG: handle any undo emoji reaction requests')
    outbox_undo_reaction(recent_posts_cache,
                         base_dir, http_prefix,
                         postToNickname, domain, port,
                         message_json, debug)

    if debug:
        print('DEBUG: handle any undo announce requests')
    outbox_undo_announce(recent_posts_cache,
                         base_dir, http_prefix,
                         postToNickname, domain, port,
                         message_json, debug)

    if debug:
        print('DEBUG: handle any bookmark requests')
    outbox_bookmark(recent_posts_cache,
                    base_dir, http_prefix,
                    postToNickname, domain, port,
                    message_json, debug)
    if debug:
        print('DEBUG: handle any undo bookmark requests')
    outbox_undo_bookmark(recent_posts_cache,
                         base_dir, http_prefix,
                         postToNickname, domain, port,
                         message_json, debug)

    if debug:
        print('DEBUG: handle delete requests')
    outbox_delete(base_dir, http_prefix,
                  postToNickname, domain,
                  message_json, debug,
                  allow_deletion,
                  recent_posts_cache)

    if debug:
        print('DEBUG: handle block requests')
    outbox_block(base_dir, http_prefix,
                 postToNickname, domain,
                 port,
                 message_json, debug)

    if debug:
        print('DEBUG: handle undo block requests')
    outbox_undo_block(base_dir, http_prefix,
                      postToNickname, domain,
                      port, message_json, debug)

    if debug:
        print('DEBUG: handle mute requests')
    outbox_mute(base_dir, http_prefix,
                postToNickname, domain,
                port,
                message_json, debug,
                recent_posts_cache)

    if debug:
        print('DEBUG: handle undo mute requests')
    outbox_undo_mute(base_dir, http_prefix,
                     postToNickname, domain,
                     port,
                     message_json, debug,
                     recent_posts_cache)

    if debug:
        print('DEBUG: handle share uploads')
    outbox_share_upload(base_dir, http_prefix, postToNickname, domain,
                        port, message_json, debug, city,
                        system_language, translate, low_bandwidth,
                        content_license_url)

    if debug:
        print('DEBUG: handle undo share uploads')
    outbox_undo_share_upload(base_dir, http_prefix,
                             postToNickname, domain,
                             port, message_json, debug)

    if debug:
        print('DEBUG: handle actor updates from c2s')
    _person_receive_update_outbox(recent_posts_cache,
                                  base_dir, http_prefix,
                                  postToNickname, domain, port,
                                  message_json, debug)

    if debug:
        print('DEBUG: sending c2s post to named addresses')
        if message_json.get('to'):
            print('c2s sender: ' +
                  postToNickname + '@' + domain + ':' + str(port) +
                  ' recipient: ' + str(message_json['to']))
        else:
            print('c2s sender: ' +
                  postToNickname + '@' + domain + ':' + str(port))
    namedAddressesThread = \
        send_to_named_addresses_thread(server.session, base_dir,
                                       postToNickname,
                                       domain, onion_domain, i2p_domain, port,
                                       http_prefix,
                                       federation_list,
                                       send_threads,
                                       postLog,
                                       cached_webfingers,
                                       person_cache,
                                       message_json, debug,
                                       version,
                                       shared_items_federated_domains,
                                       sharedItemFederationTokens,
                                       signing_priv_key_pem)
    followers_threads.append(namedAddressesThread)
    return True