epicyon/posts.py

__filename__ = "posts.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.5.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "ActivityPub"

import json
import html
import datetime
import os
import shutil
import sys
import time
import random
from time import gmtime, strftime
from collections import OrderedDict
from threads import thread_with_trace
from threads import begin_thread
from cache import get_actor_public_key_from_id
from cache import store_person_in_cache
from cache import get_person_from_cache
from cache import expire_person_cache
from pprint import pprint
from session import create_session
from session import get_json
from session import post_json
from session import post_json_string
from session import post_image
from session import get_json_valid
from webfinger import webfinger_handle
from httpsig import create_signed_header
from siteactive import site_is_active
from languages import understood_post_language
from utils import string_contains
from utils import get_post_attachments
from utils import is_premium_account
from utils import contains_private_key
from utils import get_url_from_post
from utils import date_from_string_format
from utils import date_epoch
from utils import date_utcnow
from utils import get_attributed_to
from utils import contains_statuses
from utils import contains_invalid_actor_url_chars
from utils import acct_handle_dir
from utils import is_dm
from utils import remove_eol
from utils import text_in_file
from utils import get_media_descriptions_from_post
from utils import valid_hash_tag
from utils import get_audio_extensions
from utils import get_summary_from_post
from utils import get_user_paths
from utils import invalid_ciphertext
from utils import has_object_string_type
from utils import remove_id_ending
from utils import replace_users_with_at
from utils import has_group_type
from utils import get_base_content_from_post
from utils import remove_domain_port
from utils import get_port_from_domain
from utils import has_object_dict
from utils import reject_post_id
from utils import remove_invalid_chars
from utils import file_last_modified
from utils import is_public_post
from utils import has_users_path
from utils import valid_post_date
from utils import get_full_domain
from utils import get_followers_list
from utils import is_evil
from utils import get_status_number
from utils import create_person_dir
from utils import url_permitted
from utils import get_nickname_from_actor
from utils import get_domain_from_actor
from utils import delete_post
from utils import valid_nickname
from utils import locate_post
from utils import load_json
from utils import save_json
from utils import get_config_param
from utils import locate_news_votes
from utils import locate_news_arrival
from utils import votes_on_newswire_item
from utils import remove_html
from utils import dangerous_markup
from utils import acct_dir
from utils import local_actor_url
from utils import get_reply_to
from utils import get_actor_from_post
from media import get_music_metadata
from media import attach_media
from media import replace_you_tube
from media import replace_twitter
from content import reject_twitter_summary
from content import words_similarity
from content import limit_repeated_words
from content import post_tag_exists
from content import remove_long_words
from content import add_html_tags
from content import replace_emoji_from_tags
from content import remove_text_formatting
from content import add_auto_cw
from auth import create_basic_auth_header
from blocking import is_blocked_hashtag
from blocking import is_blocked
from blocking import is_blocked_domain
from filters import is_filtered
from filters import is_question_filtered
from git import convert_post_to_patch
from linked_data_sig import generate_json_signature
from petnames import resolve_petnames
from video import convert_video_to_note
from context import get_individual_post_context
from maps import get_location_dict_from_tags
from maps import geocoords_from_map_link
from keys import get_person_key
from markdown import markdown_to_html
from followerSync import update_followers_sync_cache
from question import is_question


def convert_post_content_to_html(message_json: {}) -> None:
    """Convert post content to html
    """
    obj_json = message_json
    if has_object_dict(message_json):
        obj_json = message_json['object']
    if not obj_json.get('mediaType'):
        return
    if not obj_json.get('content'):
        return
    if obj_json['mediaType'] == 'text/markdown':
        content_str = obj_json['content']
        obj_json['content'] = markdown_to_html(content_str)
        obj_json['mediaType'] = 'text/html'
        if obj_json.get('contentMap'):
            langs_dict = obj_json['contentMap']
            if isinstance(langs_dict, dict):
                for lang, content_str in langs_dict.items():
                    if not isinstance(content_str, str):
                        continue
                    obj_json['contentMap'][lang] = \
                        markdown_to_html(content_str)


def is_moderator(base_dir: str, nickname: str) -> bool:
    """Returns true if the given nickname is a moderator
    """
    moderators_file = base_dir + '/accounts/moderators.txt'

    if not os.path.isfile(moderators_file):
        admin_name = get_config_param(base_dir, 'admin')
        if not admin_name:
            return False
        if admin_name == nickname:
            return True
        return False

    with open(moderators_file, 'r', encoding='utf-8') as fp_mod:
        lines = fp_mod.readlines()
        if len(lines) == 0:
            admin_name = get_config_param(base_dir, 'admin')
            if not admin_name:
                return False
            if admin_name == nickname:
                return True
        for moderator in lines:
            moderator = moderator.strip('\n').strip('\r')
            if moderator == nickname:
                return True
    return False


def no_of_followers_on_domain(base_dir: str, handle: str,
                              domain: str, follow_file='followers.txt') -> int:
    """Returns the number of followers of the given handle from the
    given domain
    """
    filename = acct_handle_dir(base_dir, handle) + '/' + follow_file
    if not os.path.isfile(filename):
        return 0

    ctr = 0
    with open(filename, 'r', encoding='utf-8') as followers_file:
        for follower_handle in followers_file:
            if '@' in follower_handle:
                follower_domain = follower_handle.split('@')[1]
                follower_domain = remove_eol(follower_domain)
                if domain == follower_domain:
                    ctr += 1
    return ctr


def _clean_html(raw_html: str) -> str:
    # text=BeautifulSoup(raw_html, 'html.parser').get_text()
    text = raw_html
    return html.unescape(text)


def get_user_url(wf_request: {}, source_id: int, debug: bool) -> str:
    """Gets the actor url from a webfinger request
    """
    if not wf_request.get('links'):
        if source_id == 72367:
            print('get_user_url ' + str(source_id) +
                  ' failed to get display name for webfinger ' +
                  str(wf_request))
        else:
            print('get_user_url webfinger activity+json contains no links ' +
                  str(source_id) + ' ' + str(wf_request))
        return None
    for link in wf_request['links']:
        if not (link.get('type') and link.get('href')):
            continue
        if 'application/activity+json' not in link['type'] and \
           'application/ld+json' not in link['type']:
            continue
        if '/@' not in link['href']:
            if debug and not has_users_path(link['href']):
                print('get_user_url webfinger activity+json ' +
                      'contains single user instance actor ' +
                      str(source_id) + ' ' + str(link))
        url = link['href']
        if not contains_invalid_actor_url_chars(url):
            return remove_html(url)
    return None


def parse_user_feed(signing_priv_key_pem: str,
                    session, feed_url: str, as_header: {},
                    project_version: str, http_prefix: str,
                    origin_domain: str, debug: bool, depth: int = 0) -> []:
    """Returns a feed of posts for an account url
    """
    if depth > 10:
        if debug:
            print('Maximum search depth reached')
        return None

    if debug:
        print('Getting user feed for ' + feed_url)
        print('User feed header ' + str(as_header))
        print('http_prefix ' + str(http_prefix))
        print('origin_domain ' + str(origin_domain))

    feed_json = \
        get_json(signing_priv_key_pem, session, feed_url, as_header, None,
                 debug, project_version, http_prefix, origin_domain)
    if not get_json_valid(feed_json):
        profile_str = 'https://www.w3.org/ns/activitystreams'
        accept_str = 'application/ld+json; profile="' + profile_str + '"'
        if as_header['Accept'] != accept_str:
            as_header = {
                'Accept': accept_str
            }
            feed_json = get_json(signing_priv_key_pem, session, feed_url,
                                 as_header, None, debug, project_version,
                                 http_prefix, origin_domain)
    if not get_json_valid(feed_json):
        if debug:
            print('No user feed was returned')
        return None

    if debug:
        print('User feed:')
        pprint(feed_json)

    if 'orderedItems' in feed_json:
        return feed_json['orderedItems']
    if 'items' in feed_json:
        return feed_json['items']

    next_url = None
    if 'first' in feed_json:
        next_url = feed_json['first']
    elif 'next' in feed_json:
        next_url = feed_json['next']

    if debug:
        print('User feed next url: ' + str(next_url))

    if next_url:
        if isinstance(next_url, str):
            if '?max_id=0' not in next_url:
                user_feed = \
                    parse_user_feed(signing_priv_key_pem,
                                    session, next_url, as_header,
                                    project_version, http_prefix,
                                    origin_domain, debug, depth + 1)
                if user_feed:
                    return user_feed
        elif isinstance(next_url, dict):
            user_feed = next_url
            if user_feed.get('orderedItems'):
                return user_feed['orderedItems']
            if user_feed.get('items'):
                return user_feed['items']
    return None


def _get_person_box_actor(session, base_dir: str, actor: str,
                          profile_str: str, as_header: {},
                          debug: bool, project_version: str,
                          http_prefix: str, origin_domain: str,
                          person_cache: {},
                          signing_priv_key_pem: str,
                          source_id: int) -> {}:
    """Returns the actor json for the given actor url
    """
    person_json = \
        get_person_from_cache(base_dir, actor, person_cache)
    if person_json:
        return person_json

    if '/channel/' in actor or '/accounts/' in actor:
        as_header = {
            'Accept': 'application/ld+json; profile="' + profile_str + '"'
        }
    person_json = \
        get_json(signing_priv_key_pem, session, actor, as_header, None,
                 debug, project_version, http_prefix, origin_domain)
    if get_json_valid(person_json):
        return person_json
    as_header = {
        'Accept': 'application/ld+json; profile="' + profile_str + '"'
    }
    person_json = \
        get_json(signing_priv_key_pem, session, actor, as_header, None,
                 debug, project_version, http_prefix, origin_domain)
    if get_json_valid(person_json):
        return person_json
    print('Unable to get actor for ' + actor + ' ' + str(source_id))
    if not signing_priv_key_pem:
        print('No signing key provided when getting actor')
    return None


def get_person_box(signing_priv_key_pem: str, origin_domain: str,
                   base_dir: str, session, wf_request: {}, person_cache: {},
                   project_version: str, http_prefix: str,
                   nickname: str, domain: str,
                   box_name: str, source_id: int,
                   system_language: str) -> (str, str, str, str, str,
                                             str, str, bool):
    debug = False
    profile_str = 'https://www.w3.org/ns/activitystreams'
    as_header = {
        'Accept': 'application/activity+json; profile="' + profile_str + '"'
    }
    if not wf_request:
        print('No webfinger given')
        return None, None, None, None, None, None, None, None

    # get the actor / person_url
    if not wf_request.get('errors'):
        # get the actor url from webfinger links
        person_url = get_user_url(wf_request, source_id, debug)
    else:
        if nickname == 'dev':
            # try single user instance
            print('get_person_box: Trying single user instance with ld+json')
            person_url = http_prefix + '://' + domain
            as_header = {
                'Accept': 'application/ld+json; profile="' + profile_str + '"'
            }
        else:
            # the final fallback is a mastodon style url
            person_url = local_actor_url(http_prefix, nickname, domain)
    if not person_url:
        return None, None, None, None, None, None, None, None

    # get the actor json from the url
    person_json = \
        _get_person_box_actor(session, base_dir, person_url,
                              profile_str, as_header,
                              debug, project_version,
                              http_prefix, origin_domain,
                              person_cache, signing_priv_key_pem,
                              source_id)
    if not person_json:
        return None, None, None, None, None, None, None, None

    is_group = False
    if person_json.get('type'):
        if person_json['type'] == 'Group':
            is_group = True

    # get the url for the box/collection
    box_json = None
    if not person_json.get(box_name):
        if person_json.get('endpoints'):
            if person_json['endpoints'].get(box_name):
                box_json = person_json['endpoints'][box_name]
    else:
        box_json = person_json[box_name]
    if not box_json:
        return None, None, None, None, None, None, None, None

    person_id = None
    if person_json.get('id'):
        person_id = person_json['id']
    pub_key, pub_key_id = get_actor_public_key_from_id(person_json, None)
    shared_inbox = None
    if person_json.get('sharedInbox'):
        shared_inbox = person_json['sharedInbox']
    else:
        if person_json.get('endpoints'):
            if person_json['endpoints'].get('sharedInbox'):
                shared_inbox = person_json['endpoints']['sharedInbox']
    avatar_url = None
    if person_json.get('icon'):
        icon_dict = person_json['icon']
        if isinstance(icon_dict, list):
            if len(icon_dict) > 0:
                icon_dict = icon_dict[-1]
        if isinstance(icon_dict, dict):
            if icon_dict.get('url'):
                url_str = get_url_from_post(icon_dict['url'])
                avatar_url = remove_html(url_str)
    display_name = None
    possible_display_name = None
    if person_json.get('name'):
        possible_display_name = person_json['name']
    if person_json.get('nameMap'):
        if isinstance(person_json['nameMap'], dict):
            if system_language in person_json['nameMap']:
                if isinstance(person_json['nameMap'][system_language], str):
                    possible_display_name = \
                        person_json['nameMap'][system_language]
    if possible_display_name:
        display_name = possible_display_name
        if dangerous_markup(possible_display_name, False, []):
            display_name = '*ADVERSARY*'
        elif is_filtered(base_dir,
                         nickname, domain,
                         possible_display_name, 'en'):
            display_name = '*FILTERED*'
        # have they moved?
        if person_json.get('movedTo'):
            display_name += ' ⌂'

    store_person_in_cache(base_dir, person_url, person_json,
                          person_cache, True)

    return box_json, pub_key_id, pub_key, person_id, shared_inbox, \
        avatar_url, display_name, is_group


def _is_public_feed_post(item: {}, person_posts: {}, debug: bool) -> bool:
    """Is the given post a public feed post?
    """
    if not isinstance(item, dict):
        if debug:
            print('item object is not a dict')
            pprint(item)
        return False
    if not item.get('id'):
        if debug:
            print('No id')
        return False
    if not item.get('type'):
        if debug:
            print('No type')
        return False
    allowed_post_types = ('Create', 'Announce', 'Page', 'Note', 'Event')
    if item['type'] not in allowed_post_types:
        if debug:
            print('Not a Create/Note/Announce type')
        return False
    if item.get('object'):
        if isinstance(item['object'], dict):
            if not item['object'].get('published'):
                if debug:
                    print('No published attribute 1')
                return False
        elif isinstance(item['object'], str):
            if item['type'] != 'Announce' and \
               not item.get('published'):
                if debug:
                    print('No published attribute 2')
                return False
        else:
            if debug:
                print('object is not a dict or string')
            return False
    elif item['type'] in ('Note', 'Event', 'Page'):
        if not item.get('published'):
            if debug:
                print('No published attribute 3')
            return False
    if not person_posts.get(item['id']):
        this_item = item
        if item.get('object'):
            this_item = item['object']
        # check that this is a public post
        # #Public should appear in the "to" list
        item_is_note = False
        if item['type'] in ('Note', 'Event', 'Page'):
            item_is_note = True

        if isinstance(this_item, dict):
            if this_item.get('to'):
                is_public = False
                if isinstance(this_item['to'], list):
                    for recipient in this_item['to']:
                        if recipient.endswith('#Public') or \
                           recipient == 'as:Public' or \
                           recipient == 'Public':
                            is_public = True
                            break
                elif isinstance(this_item['to'], str):
                    recipient = this_item['to']
                    if recipient.endswith('#Public') or \
                       recipient == 'as:Public' or \
                       recipient == 'Public':
                        is_public = True
                if not is_public:
                    return False
        elif isinstance(this_item, str) or item_is_note:
            if item.get('to'):
                is_public = False
                if isinstance(item['to'], list):
                    for recipient in item['to']:
                        if recipient.endswith('#Public') or \
                           recipient == 'as:Public' or \
                           recipient == 'Public':
                            is_public = True
                            break
                elif isinstance(item['to'], str):
                    recipient = item['to']
                    if recipient.endswith('#Public') or \
                       recipient == 'as:Public' or \
                       recipient == 'Public':
                        is_public = True
                if not is_public:
                    return False
    return True


def is_create_inside_announce(item: {}) -> bool:
    """ is this a Create inside of an Announce?
    eg. lemmy feed item
    """
    if not isinstance(item, dict):
        return False
    if not item.get('type'):
        return False
    if item['type'] != 'Announce':
        return False
    if not item.get('object'):
        return False
    if not isinstance(item['object'], dict):
        return False
    if not item['object'].get('type'):
        return False
    if item['object']['type'] != 'Create':
        return False
    return True


def _get_posts(session, outbox_url: str, max_posts: int,
               max_mentions: int,
               max_emoji: int, max_attachments: int,
               federation_list: [], raw: bool,
               simple: bool, debug: bool,
               project_version: str, http_prefix: str,
               origin_domain: str, system_language: str,
               signing_priv_key_pem: str) -> {}:
    """Gets public posts from an outbox
    """
    if debug:
        print('Getting outbox posts for ' + outbox_url)
    person_posts = {}
    if not outbox_url:
        return person_posts
    profile_str = 'https://www.w3.org/ns/activitystreams'
    accept_str = \
        'application/activity+json; ' + \
        'profile="' + profile_str + '"'
    as_header = {
        'Accept': accept_str
    }
    if '/outbox/' in outbox_url:
        accept_str = \
            'application/ld+json; ' + \
            'profile="' + profile_str + '"'
        as_header = {
            'Accept': accept_str
        }
    if raw:
        if debug:
            print('Returning the raw feed')
        result = []
        i = 0
        user_feed = parse_user_feed(signing_priv_key_pem,
                                    session, outbox_url, as_header,
                                    project_version, http_prefix,
                                    origin_domain, debug)
        if user_feed:
            for item in user_feed:
                result.append(item)
                i += 1
                if i == max_posts:
                    break
        pprint(result)
        return None

    if debug:
        print('Returning a human readable version of the feed')
    user_feed = parse_user_feed(signing_priv_key_pem,
                                session, outbox_url, as_header,
                                project_version, http_prefix,
                                origin_domain, debug)
    if not user_feed:
        return person_posts

    i = 0
    for item in user_feed:
        if is_create_inside_announce(item):
            item = item['object']

        if not _is_public_feed_post(item, person_posts, debug):
            continue

        this_item = item
        this_item_type = item['type']
        if this_item_type not in ('Note', 'Event', 'Page'):
            this_item = item['object']
            if isinstance(this_item, str):
                if '://' in this_item:
                    profile_str = 'https://www.w3.org/ns/activitystreams'
                    as_header2_str = 'application/ld+json; profile="' + \
                        profile_str + '"'
                    as_header2 = {
                        'Accept': as_header2_str
                    }
                    this_item = \
                        get_json(signing_priv_key_pem, session, this_item,
                                 as_header2, None, debug, __version__,
                                 http_prefix, origin_domain)
                    if not get_json_valid(this_item):
                        continue

        content = get_base_content_from_post(this_item, system_language)
        if not content:
            continue
        content = content.replace(''', "'")

        mentions = []
        emoji = {}
        summary = ''
        in_reply_to = ''
        attachment = []
        sensitive = False
        if isinstance(this_item, dict):
            if this_item.get('tag'):
                for tag_item in this_item['tag']:
                    if not tag_item.get('type'):
                        continue
                    tag_type = tag_item['type'].lower()
                    if tag_type == 'emoji':
                        if tag_item.get('name') and tag_item.get('icon'):
                            if tag_item['icon'].get('url'):
                                # No emoji from non-permitted domains
                                url_str = \
                                    get_url_from_post(tag_item['icon']['url'])
                                if url_permitted(url_str,
                                                 federation_list):
                                    emoji_name = tag_item['name']
                                    emoji_icon = remove_html(url_str)
                                    emoji[emoji_name] = emoji_icon
                                else:
                                    if debug:
                                        print('url not permitted ' + url_str)
                    if tag_type == 'mention':
                        if tag_item.get('name'):
                            if tag_item['name'] not in mentions:
                                mentions.append(tag_item['name'])
            if len(mentions) > max_mentions:
                if debug:
                    print('max mentions reached')
                continue
            if len(emoji) > max_emoji:
                if debug:
                    print('max emojis reached')
                continue

            if this_item.get('summaryMap'):
                if this_item['summaryMap'].get(system_language):
                    summary = this_item['summaryMap'][system_language]
            if not summary and this_item.get('summary'):
                if this_item['summary']:
                    summary = this_item['summary']

            reply_id = get_reply_to(this_item)
            if reply_id:
                if isinstance(reply_id, str):
                    # No replies to non-permitted domains
                    if not url_permitted(reply_id,
                                         federation_list):
                        if debug:
                            print('url not permitted ' +
                                  reply_id)
                        continue
                    in_reply_to = reply_id

            post_attachments = get_post_attachments(this_item)
            if post_attachments:
                if len(post_attachments) > max_attachments:
                    if debug:
                        print('max attachments reached')
                    continue
                if post_attachments:
                    for attach in post_attachments:
                        if attach.get('name') and attach.get('url'):
                            # no attachments from non-permitted domains
                            url_str = get_url_from_post(attach['url'])
                            attach_url = remove_html(url_str)
                            if url_permitted(attach_url,
                                             federation_list):
                                attachment.append([attach['name'],
                                                   attach_url])
                            else:
                                if debug:
                                    print('url not permitted ' + url_str)

            sensitive = False
            if this_item.get('sensitive'):
                sensitive = this_item['sensitive']

        if content:
            if simple:
                print(_clean_html(content) + '\n')
            else:
                pprint(item)
                person_posts[this_item['id']] = {
                    "sensitive": sensitive,
                    "inreplyto": in_reply_to,
                    "summary": summary,
                    "html": content,
                    "plaintext": _clean_html(content),
                    "attachment": attachment,
                    "mentions": mentions,
                    "emoji": emoji
                }
            i += 1

            if i == max_posts:
                break
    return person_posts


def _get_common_words() -> str:
    """Returns a list of common words
    """
    return (
        'that', 'some', 'about', 'then', 'they', 'were',
        'also', 'from', 'with', 'this', 'have', 'more',
        'need', 'here', 'would', 'these', 'into', 'very',
        'well', 'when', 'what', 'your', 'there', 'which',
        'even', 'there', 'such', 'just', 'those', 'only',
        'will', 'much', 'than', 'them', 'each', 'goes',
        'been', 'over', 'their', 'where', 'could', 'though',
        'like', 'think', 'same', 'maybe', 'really', 'thing',
        'something', 'possible', 'actual', 'actually',
        'because', 'around', 'having', 'especially', 'other',
        'making', 'made', 'make', 'makes', 'including',
        'includes', 'know', 'knowing', 'knows', 'things',
        'say', 'says', 'saying', 'many', 'somewhat',
        'problem', 'problems', 'idea', 'ideas',
        'using', 'uses', 'https', 'still', 'want', 'wants'
    )


def _update_word_frequency(content: str, word_frequency: {}) -> None:
    """Creates a dictionary containing words and the number of times
    that they appear
    """
    plain_text = remove_html(content)
    remove_chars = ('.', ';', '?', '\n', ':')
    for char in remove_chars:
        plain_text = plain_text.replace(char, ' ')
    words_list = plain_text.split(' ')
    common_words = _get_common_words()
    for word in words_list:
        word_len = len(word)
        if word_len < 3:
            continue
        if word_len < 4:
            if word.upper() != word:
                continue
        if string_contains(word, ('&', '"', '@', "'", "--", '//')):
            continue
        if word.lower() in common_words:
            continue
        if word_frequency.get(word):
            word_frequency[word] += 1
        else:
            word_frequency[word] = 1


def get_post_domains(session, outbox_url: str, max_posts: int, debug: bool,
                     project_version: str, http_prefix: str, domain: str,
                     word_frequency: {}, domain_list: [],
                     system_language: str, signing_priv_key_pem: str) -> []:
    """Returns a list of domains referenced within public posts
    """
    if not outbox_url:
        return []
    profile_str = 'https://www.w3.org/ns/activitystreams'
    accept_str = \
        'application/activity+json; ' + \
        'profile="' + profile_str + '"'
    as_header = {
        'Accept': accept_str
    }
    if '/outbox/' in outbox_url:
        accept_str = \
            'application/ld+json; ' + \
            'profile="' + profile_str + '"'
        as_header = {
            'Accept': accept_str
        }

    post_domains = domain_list

    i = 0
    user_feed = parse_user_feed(signing_priv_key_pem,
                                session, outbox_url, as_header,
                                project_version, http_prefix, domain, debug)
    if not user_feed:
        return post_domains

    for item in user_feed:
        i += 1
        if i > max_posts:
            break
        if not has_object_dict(item):
            continue
        content_str = get_base_content_from_post(item, system_language)
        if content_str:
            _update_word_frequency(content_str, word_frequency)
        reply_id = get_reply_to(item['object'])
        if reply_id:
            if isinstance(reply_id, str):
                post_domain, _ = \
                    get_domain_from_actor(reply_id)
                if post_domain:
                    if post_domain not in post_domains:
                        post_domains.append(post_domain)

        if item['object'].get('tag'):
            for tag_item in item['object']['tag']:
                if not tag_item.get('type'):
                    continue
                tag_type = tag_item['type'].lower()
                if tag_type == 'mention':
                    if tag_item.get('href'):
                        tag_url = remove_html(tag_item['href'])
                        post_domain, _ = \
                            get_domain_from_actor(tag_url)
                        if post_domain:
                            if post_domain not in post_domains:
                                post_domains.append(post_domain)
    return post_domains


def _get_posts_for_blocked_domains(base_dir: str,
                                   session, outbox_url: str, max_posts: int,
                                   debug: bool,
                                   project_version: str, http_prefix: str,
                                   domain: str,
                                   signing_priv_key_pem: str) -> {}:
    """Returns a dictionary of posts for blocked domains
    """
    if not outbox_url:
        return {}
    profile_str = 'https://www.w3.org/ns/activitystreams'
    accept_str = \
        'application/activity+json; ' + \
        'profile="' + profile_str + '"'
    as_header = {
        'Accept': accept_str
    }
    if '/outbox/' in outbox_url:
        accept_str = \
            'application/ld+json; ' + \
            'profile="' + profile_str + '"'
        as_header = {
            'Accept': accept_str
        }

    blocked_posts = {}

    i = 0
    user_feed = parse_user_feed(signing_priv_key_pem,
                                session, outbox_url, as_header,
                                project_version, http_prefix, domain, debug)
    if not user_feed:
        return blocked_posts

    for item in user_feed:
        i += 1
        if i > max_posts:
            break
        if not has_object_dict(item):
            continue
        reply_id = get_reply_to(item['object'])
        if reply_id:
            if isinstance(reply_id, str):
                post_domain, _ = \
                    get_domain_from_actor(reply_id)
                if not post_domain:
                    continue
                if is_blocked_domain(base_dir, post_domain, None, None):
                    if item['object'].get('url'):
                        url = get_url_from_post(item['object']['url'])
                    else:
                        url = get_url_from_post(item['object']['id'])
                    url = remove_html(url)
                    if not blocked_posts.get(post_domain):
                        blocked_posts[post_domain] = [url]
                    else:
                        if url not in blocked_posts[post_domain]:
                            blocked_posts[post_domain].append(url)

        if item['object'].get('tag'):
            for tag_item in item['object']['tag']:
                if not tag_item.get('type'):
                    continue
                tag_type = tag_item['type'].lower()
                if tag_type == 'mention' and tag_item.get('href'):
                    tag_url = remove_html(tag_item['href'])
                    post_domain, _ = \
                        get_domain_from_actor(tag_url)
                    if not post_domain:
                        continue
                    if is_blocked_domain(base_dir, post_domain, None, None):
                        if item['object'].get('url'):
                            url = get_url_from_post(item['object']['url'])
                        else:
                            url = get_url_from_post(item['object']['id'])
                        url = remove_html(url)
                        if not blocked_posts.get(post_domain):
                            blocked_posts[post_domain] = [url]
                        else:
                            if url not in blocked_posts[post_domain]:
                                blocked_posts[post_domain].append(url)
    return blocked_posts


def delete_all_posts(base_dir: str,
                     nickname: str, domain: str, boxname: str) -> None:
    """Deletes all posts for a person from inbox or outbox
    """
    if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews'):
        return
    box_dir = create_person_dir(nickname, domain, base_dir, boxname)
    for delete_filename in os.scandir(box_dir):
        delete_filename = delete_filename.name
        file_path = os.path.join(box_dir, delete_filename)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path, ignore_errors=False, onerror=None)
        except OSError as ex:
            print('ERROR: delete_all_posts ' + str(ex))


def _save_last_published(base_dir: str, nickname: str, domain: str,
                         published: str) -> None:
    """Saves the last published date for the outbox
    """
    if not published:
        return

    published_filename = \
        acct_dir(base_dir, nickname, domain) + '/.last_published'
    try:
        with open(published_filename, 'w+', encoding='utf-8') as fp_last:
            fp_last.write(published)
    except OSError:
        print('EX: unable to save last published time ' +
              published_filename)


def save_post_to_box(base_dir: str, http_prefix: str, post_id: str,
                     nickname: str, domain: str, post_json_object: {},
                     boxname: str) -> str:
    """Saves the give json to the give box
    Returns the filename
    """
    if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews', 'scheduled'):
        return None
    original_domain = domain
    domain = remove_domain_port(domain)

    if not post_id:
        status_number, _ = get_status_number()
        post_id = \
            local_actor_url(http_prefix, nickname, original_domain) + \
            '/statuses/' + status_number
        post_json_object['id'] = post_id + '/activity'
    published = None
    if post_json_object.get('published'):
        published = post_json_object['published']
    if has_object_dict(post_json_object):
        post_json_object['object']['id'] = post_id
        post_json_object['object']['atomUri'] = post_id
        if post_json_object['object'].get('published'):
            published = post_json_object['object']['published']

    box_dir = create_person_dir(nickname, domain, base_dir, boxname)
    filename = box_dir + '/' + post_id.replace('/', '#') + '.json'

    save_json(post_json_object, filename)
    # if this is an outbox post with a duplicate in the inbox then save to both
    # This happens for edited posts
    if '/outbox/' in filename:
        _save_last_published(base_dir, nickname, domain, published)

        inbox_filename = filename.replace('/outbox/', '/inbox/')
        if os.path.isfile(inbox_filename):
            save_json(post_json_object, inbox_filename)
            base_filename = \
                filename.replace('/outbox/',
                                 '/postcache/').replace('.json', '')
            ssml_filename = base_filename + '.ssml'
            if os.path.isfile(ssml_filename):
                try:
                    os.remove(ssml_filename)
                except OSError:
                    print('EX: save_post_to_box unable to delete ssml file ' +
                          ssml_filename)
            html_filename = base_filename + '.html'
            if os.path.isfile(html_filename):
                try:
                    os.remove(html_filename)
                except OSError:
                    print('EX: save_post_to_box unable to delete html file ' +
                          html_filename)
    return filename


def _update_hashtags_index(base_dir: str, tag: {}, new_post_id: str,
                           nickname: str) -> None:
    """Writes the post url for hashtags to a file
    This allows posts for a hashtag to be quickly looked up
    """
    if tag['type'] != 'Hashtag':
        return

    # create hashtags directory
    tags_dir = base_dir + '/tags'
    if not os.path.isdir(tags_dir):
        os.mkdir(tags_dir)
    tag_name = tag['name']
    tags_filename = tags_dir + '/' + tag_name[1:] + '.txt'

    new_post_id = new_post_id.replace('/', '#')

    if not os.path.isfile(tags_filename):
        days_diff = date_utcnow() - date_epoch()
        days_since_epoch = days_diff.days
        tag_line = \
            str(days_since_epoch) + '  ' + nickname + '  ' + \
            new_post_id + '\n'
        # create a new tags index file
        try:
            with open(tags_filename, 'w+', encoding='utf-8') as tags_file:
                tags_file.write(tag_line)
        except OSError:
            print('EX: _update_hashtags_index unable to write tags file ' +
                  tags_filename)
    else:
        # prepend to tags index file
        if not text_in_file(new_post_id, tags_filename):
            days_diff = date_utcnow() - date_epoch()
            days_since_epoch = days_diff.days
            tag_line = \
                str(days_since_epoch) + '  ' + nickname + '  ' + \
                new_post_id + '\n'
            try:
                with open(tags_filename, 'r+', encoding='utf-8') as tags_file:
                    content = tags_file.read()
                    if tag_line not in content:
                        tags_file.seek(0, 0)
                        tags_file.write(tag_line + content)
            except OSError as ex:
                print('EX: Failed to write entry to tags file ' +
                      tags_filename + ' ' + str(ex))


def _add_schedule_post(base_dir: str, nickname: str, domain: str,
                       event_date_str: str, post_id: str) -> None:
    """Adds a scheduled post to the index
    """
    handle = nickname + '@' + domain
    schedule_index_filename = \
        acct_handle_dir(base_dir, handle) + '/schedule.index'

    index_str = event_date_str + ' ' + post_id.replace('/', '#')
    if os.path.isfile(schedule_index_filename):
        if not text_in_file(index_str, schedule_index_filename):
            try:
                with open(schedule_index_filename, 'r+',
                          encoding='utf-8') as schedule_file:
                    content = schedule_file.read()
                    if index_str + '\n' not in content:
                        schedule_file.seek(0, 0)
                        schedule_file.write(index_str + '\n' + content)
                        print('DEBUG: scheduled post added to index')
            except OSError as ex:
                print('EX: Failed to write entry to scheduled posts index ' +
                      schedule_index_filename + ' ' + str(ex))
    else:
        try:
            with open(schedule_index_filename, 'w+',
                      encoding='utf-8') as schedule_file:
                schedule_file.write(index_str + '\n')
        except OSError as ex:
            print('EX: Failed to write entry to scheduled posts index2 ' +
                  schedule_index_filename + ' ' + str(ex))


def valid_content_warning(summary: str) -> str:
    """Returns a validated content warning
    """
    cw_str = remove_html(summary)
    # hashtags within content warnings apparently cause a lot of trouble
    # so remove them
    if '#' in cw_str:
        cw_str = cw_str.replace('#', '').replace('  ', ' ')
    return remove_invalid_chars(cw_str)


def _create_post_cw_from_reply(base_dir: str, nickname: str, domain: str,
                               in_reply_to: str,
                               sensitive: bool, summary: str,
                               system_language: str,
                               languages_understood: []) -> (bool, str):
    """If this is a reply and the original post has a CW
    then use the same CW
    """
    reply_to_json = None
    if in_reply_to and not sensitive:
        # locate the post which this is a reply to and check if
        # it has a content warning. If it does then reproduce
        # the same warning
        reply_post_filename = \
            locate_post(base_dir, nickname, domain, in_reply_to)
        if reply_post_filename:
            reply_to_json = load_json(reply_post_filename)
    if reply_to_json:
        if reply_to_json.get('object'):
            if reply_to_json['object'].get('sensitive'):
                if reply_to_json['object']['sensitive']:
                    sensitive = True
                    if reply_to_json['object'].get('summary'):
                        summary = \
                            get_summary_from_post(reply_to_json,
                                                  system_language,
                                                  languages_understood)

    return sensitive, summary


def _attach_post_license(post_json_object: {},
                         content_license_url: str) -> None:
    """Attaches a license to each post
    Also see:
    https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-c118.md
    NOTE: at present (Jan 2023) there is no consensus about how to
    represent license information on ActivityPub posts, so this might
    need to change if such a consensus appears.
    """
    if not content_license_url:
        return
    post_json_object['attachment'].append({
        "type": "PropertyValue",
        "name": "license",
        "value": content_license_url
    })
    post_json_object['schema:license'] = content_license_url
    post_json_object['license'] = content_license_url


def _attach_buy_link(post_json_object: {},
                     buy_url: str, translate: {}) -> None:
    """Attaches a link for buying something
    """
    if not buy_url:
        return
    if '://' not in buy_url:
        return
    if ' ' in buy_url or '<' in buy_url:
        return
    buy_str = 'Buy'
    if translate.get(buy_str):
        buy_str = translate[buy_str]
    if 'attachment' not in post_json_object:
        post_json_object['attachment'] = []
    post_json_object['attachment'].append({
        "type": "Link",
        "name": buy_str,
        "href": buy_url,
        "rel": "payment",
        "mediaType": "text/html"
    })


def _attach_chat_link(post_json_object: {},
                      chat_url: str) -> None:
    """Attaches a chat link
    """
    if not chat_url:
        return
    if '://' not in chat_url:
        return
    if ' ' in chat_url or '<' in chat_url:
        return
    if 'attachment' not in post_json_object:
        post_json_object['attachment'] = []
    post_json_object['attachment'].append({
        "type": "Link",
        "name": "Chat",
        "href": chat_url,
        "rel": "discussion"
    })


def _create_post_s2s(base_dir: str, nickname: str, domain: str, port: int,
                     http_prefix: str, content: str, status_number: str,
                     published: str, new_post_id: str, post_context: {},
                     to_recipients: [], to_cc: [], in_reply_to: str,
                     sensitive: bool, comments_enabled: bool,
                     tags: [], attach_image_filename: str,
                     media_type: str, image_description: str,
                     video_transcript: str, city: str,
                     post_object_type: str, summary: str,
                     in_reply_to_atom_uri: str, system_language: str,
                     conversation_id: str, low_bandwidth: bool,
                     content_license_url: str,
                     media_license_url: str, media_creator: str,
                     buy_url: str, chat_url: str, translate: {}) -> {}:
    """Creates a new server-to-server post
    """
    actor_url = local_actor_url(http_prefix, nickname, domain)
    id_str = \
        local_actor_url(http_prefix, nickname, domain) + \
        '/statuses/' + status_number + '/replies'
    new_post_url = \
        http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
    new_post_attributed_to = \
        local_actor_url(http_prefix, nickname, domain)
    if not conversation_id:
        conversation_id = new_post_id
    new_post = {
        '@context': post_context,
        'id': new_post_id + '/activity',
        'type': 'Create',
        'actor': actor_url,
        'published': published,
        'to': to_recipients,
        'cc': to_cc,
        'object': {
            'id': new_post_id,
            'conversation': conversation_id,
            'context': conversation_id,
            'type': post_object_type,
            'summary': summary,
            'inReplyTo': in_reply_to,
            'published': published,
            'url': new_post_url,
            'attributedTo': new_post_attributed_to,
            'to': to_recipients,
            'cc': to_cc,
            'sensitive': sensitive,
            'atomUri': new_post_id,
            'inReplyToAtomUri': in_reply_to_atom_uri,
            'commentsEnabled': comments_enabled,
            'rejectReplies': not comments_enabled,
            'mediaType': 'text/html',
            'content': content,
            'contentMap': {
                system_language: content
            },
            'attachment': [],
            'tag': tags,
            'replies': {
                'id': id_str,
                'type': 'Collection',
                'first': {
                    'type': 'CollectionPage',
                    'next': id_str + '?only_other_accounts=true&page=true',
                    'partOf': id_str,
                    'items': []
                }
            },
            "crawlable": False
        }
    }

    # pixelfed/friendica style location representation
    location = get_location_dict_from_tags(tags)
    if location:
        if location.get('name') and \
           location.get('longitude') and \
           location.get('latitude'):
            new_post['object']['location'] = {
                'type': 'Place',
                'name': location['name'],
                'longitude': location['longitude'],
                'latitude': location['latitude']
            }
        elif location.get('name'):
            new_post['object']['location'] = {
                'type': 'Place',
                'name': location['name']
            }

    if attach_image_filename:
        new_post['object'] = \
            attach_media(base_dir, http_prefix, nickname, domain, port,
                         new_post['object'], attach_image_filename,
                         media_type, image_description, video_transcript,
                         city, low_bandwidth,
                         media_license_url, media_creator, system_language)
    _attach_post_license(new_post['object'], content_license_url)
    _attach_buy_link(new_post['object'], buy_url, translate)
    _attach_chat_link(new_post['object'], chat_url)
    return new_post


def _create_post_c2s(base_dir: str, nickname: str, domain: str, port: int,
                     http_prefix: str, content: str, status_number: str,
                     published: str, new_post_id: str, post_context: {},
                     to_recipients: [], to_cc: [], in_reply_to: str,
                     sensitive: bool, comments_enabled: bool,
                     tags: [], attach_image_filename: str,
                     media_type: str, image_description: str,
                     video_transcript: str, city: str,
                     post_object_type: str, summary: str,
                     in_reply_to_atom_uri: str, system_language: str,
                     conversation_id: str, low_bandwidth: str,
                     content_license_url: str, media_license_url: str,
                     media_creator: str, buy_url: str, chat_url: str,
                     translate: {}) -> {}:
    """Creates a new client-to-server post
    """
    domain_full = get_full_domain(domain, port)
    id_str = \
        local_actor_url(http_prefix, nickname, domain_full) + \
        '/statuses/' + status_number + '/replies'
    new_post_url = \
        http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
    if not conversation_id:
        conversation_id = new_post_id
    new_post = {
        "@context": post_context,
        'id': new_post_id,
        'conversation': conversation_id,
        'context': conversation_id,
        'type': post_object_type,
        'summary': summary,
        'inReplyTo': in_reply_to,
        'published': published,
        'url': new_post_url,
        'attributedTo': local_actor_url(http_prefix, nickname, domain_full),
        'to': to_recipients,
        'cc': to_cc,
        'sensitive': sensitive,
        'atomUri': new_post_id,
        'inReplyToAtomUri': in_reply_to_atom_uri,
        'commentsEnabled': comments_enabled,
        'rejectReplies': not comments_enabled,
        'mediaType': 'text/html',
        'content': content,
        'contentMap': {
            system_language: content
        },
        'attachment': [],
        'tag': tags,
        'replies': {
            'id': id_str,
            'type': 'Collection',
            'first': {
                'type': 'CollectionPage',
                'next': id_str + '?only_other_accounts=true&page=true',
                'partOf': id_str,
                'items': []
            }
        },
        "crawlable": False
    }

    # pixelfed/friendica style location representation
    location = get_location_dict_from_tags(tags)
    if location:
        if location.get('name') and \
           location.get('longitude') and \
           location.get('latitude'):
            new_post['location'] = {
                'type': 'Place',
                'name': location['name'],
                'longitude': location['longitude'],
                'latitude': location['latitude']
            }
        elif location.get('name'):
            new_post['location'] = {
                'type': 'Place',
                'name': location['name']
            }

    if attach_image_filename:
        new_post = \
            attach_media(base_dir, http_prefix, nickname, domain, port,
                         new_post, attach_image_filename,
                         media_type, image_description, video_transcript,
                         city, low_bandwidth,
                         media_license_url, media_creator,
                         system_language)
    _attach_post_license(new_post, content_license_url)
    _attach_buy_link(new_post, buy_url, translate)
    _attach_chat_link(new_post, chat_url)
    return new_post


def _create_post_place_and_time(event_date: str, end_date: str,
                                event_time: str, end_time: str,
                                summary: str, content: str,
                                schedule_post: bool,
                                event_uuid: str,
                                location: str,
                                tags: []) -> str:
    """Adds a place and time to the tags on a new post
    """
    end_date_str = None
    if end_date:
        event_name = summary
        if not event_name:
            event_name = content
        end_date_str = end_date
        if end_time:
            if end_time.endswith('Z'):
                end_date_str = end_date + 'T' + end_time
            else:
                end_date_str = end_date + 'T' + end_time + \
                    ':00' + strftime("%z", gmtime())
        else:
            if not event_time:
                end_date_str = end_date + 'T12:00:00Z'
            else:
                if event_time.endswith('Z'):
                    end_date_str = end_date + 'T' + event_time
                else:
                    end_date_str = \
                        end_date + 'T' + event_time + ':00' + \
                        strftime("%z", gmtime())

    # get the starting date and time
    event_date_str = None
    if event_date:
        event_name = summary
        if not event_name:
            event_name = content
        event_date_str = event_date
        if event_time:
            if event_time.endswith('Z'):
                event_date_str = event_date + 'T' + event_time
            else:
                event_date_str = event_date + 'T' + event_time + \
                    ':00' + strftime("%z", gmtime())
        else:
            event_date_str = event_date + 'T12:00:00Z'
        if not end_date_str:
            end_date_str = event_date_str
        if not schedule_post and not event_uuid:
            tags.append({
                "@context": "https://www.w3.org/ns/activitystreams",
                "type": "Event",
                "name": event_name,
                "startTime": event_date_str,
                "endTime": end_date_str
            })
    if location and not event_uuid:
        latitude = longitude = None
        if '://' in location:
            _, latitude, longitude = \
                geocoords_from_map_link(location, 'openstreetmap.org')
        if latitude and longitude:
            tags.append({
                "@context": "https://www.w3.org/ns/activitystreams",
                "type": "Place",
                "name": location,
                "latitude": latitude,
                "longitude": longitude
            })
        else:
            tags.append({
                "@context": "https://www.w3.org/ns/activitystreams",
                "type": "Place",
                "name": location
            })
    return event_date_str


def _consolidate_actors_list(actors_list: []) -> None:
    """ consolidate duplicated actors
    https://domain/@nick gets merged with https://domain/users/nick
    """
    possible_duplicate_actors = []
    for cc_actor in actors_list:
        if '/@' in cc_actor:
            if '/@/' not in cc_actor:
                if cc_actor not in possible_duplicate_actors:
                    possible_duplicate_actors.append(cc_actor)
    if possible_duplicate_actors:
        u_paths = get_user_paths()
        remove_actors = []
        for cc_actor in possible_duplicate_actors:
            for usr_path in u_paths:
                if '/@/' not in cc_actor:
                    cc_actor_full = cc_actor.replace('/@', usr_path)
                else:
                    cc_actor_full = cc_actor
                if cc_actor_full in actors_list:
                    if cc_actor not in remove_actors:
                        remove_actors.append(cc_actor)
                    break
        for cc_actor in remove_actors:
            actors_list.remove(cc_actor)


def _create_post_mentions(cc_url: str, new_post: {},
                          to_recipients: [], tags: []) -> None:
    """Updates mentions for a new post
    """
    if not cc_url:
        return
    if len(cc_url) == 0:
        return

    if new_post.get('object'):
        if cc_url not in new_post['object']['cc']:
            new_post['object']['cc'] = [cc_url] + new_post['object']['cc']

        # if this is a public post then include any mentions in cc
        to_cc = new_post['object']['cc']
        if len(to_recipients) != 1:
            return
        to_public_recipient = False
        if to_recipients[0].endswith('#Public') or \
           to_recipients[0] == 'as:Public' or \
           to_recipients[0] == 'Public':
            to_public_recipient = True
        if to_public_recipient and \
           cc_url.endswith('/followers'):
            for tag in tags:
                if tag['type'] != 'Mention':
                    continue
                if tag['href'] not in to_cc:
                    tag_url = remove_html(tag['href'])
                    new_post['object']['cc'].append(tag_url)

        _consolidate_actors_list(new_post['object']['cc'])
        new_post['cc'] = new_post['object']['cc']
    else:
        if cc_url not in new_post['cc']:
            new_post['cc'] = [cc_url] + new_post['cc']
        _consolidate_actors_list(['cc'])


def _create_post_mod_report(base_dir: str,
                            is_moderation_report: bool, new_post: {},
                            new_post_id: str) -> None:
    """ if this is a moderation report then add a status
    """
    if not is_moderation_report:
        return
    # add status
    if new_post.get('object'):
        new_post['object']['moderationStatus'] = 'pending'
    else:
        new_post['moderationStatus'] = 'pending'
    # save to index file
    moderation_index_file = base_dir + '/accounts/moderation.txt'
    try:
        with open(moderation_index_file, 'a+', encoding='utf-8') as mod_file:
            mod_file.write(new_post_id + '\n')
    except OSError:
        print('EX: unable to write moderation index file ' +
              moderation_index_file)


def get_actor_from_in_reply_to(in_reply_to: str) -> str:
    """Tries to get the replied to actor from the inReplyTo post id
    Note: this will not always be successful for some instance types
    """
    reply_nickname = get_nickname_from_actor(in_reply_to)
    if not reply_nickname:
        return None
    reply_actor = None
    if '/' + reply_nickname + '/' in in_reply_to:
        reply_actor = \
            in_reply_to.split('/' + reply_nickname + '/')[0] + \
            '/' + reply_nickname
    elif '#' + reply_nickname + '#' in in_reply_to:
        reply_actor = \
            in_reply_to.split('#' + reply_nickname + '#')[0] + \
            '#' + reply_nickname
        reply_actor = reply_actor.replace('#', '/')
    if not reply_actor:
        return None
    if '://' not in reply_actor:
        return None
    return reply_actor


def _create_post_base(base_dir: str,
                      nickname: str, domain: str, port: int,
                      to_url: str, cc_url: str, http_prefix: str, content: str,
                      save_to_file: bool,
                      client_to_server: bool, comments_enabled: bool,
                      attach_image_filename: str,
                      media_type: str, image_description: str,
                      video_transcript: str, city: str,
                      is_moderation_report: bool,
                      is_article: bool,
                      in_reply_to: str,
                      in_reply_to_atom_uri: str,
                      subject: str, schedule_post: bool,
                      event_date: str, event_time: str,
                      location: str,
                      event_uuid: str, category: str,
                      join_mode: str,
                      end_date: str, end_time: str,
                      maximum_attendee_capacity: int,
                      replies_moderation_option: str,
                      anonymous_participation_enabled: bool,
                      event_status: str, ticket_url: str,
                      system_language: str,
                      conversation_id: str, low_bandwidth: bool,
                      content_license_url: str,
                      media_license_url: str, media_creator: str,
                      languages_understood: [], translate: {},
                      buy_url: str, chat_url: str,
                      auto_cw_cache: {}) -> {}:
    """Creates a message
    """
    content = remove_invalid_chars(content)

    subject = add_auto_cw(base_dir, nickname, domain, subject, content,
                          auto_cw_cache)

    if nickname != 'news':
        mentioned_recipients = \
            get_mentioned_people(base_dir, http_prefix, content, domain, False)
    else:
        mentioned_recipients = ''

    # add hashtags from audio file ID3 tags, such as Artist, Album, etc
    if attach_image_filename and media_type:
        audio_types = get_audio_extensions()
        music_metadata = None
        for ext in audio_types:
            if ext in media_type:
                music_metadata = get_music_metadata(attach_image_filename)
                break
        if music_metadata:
            for audio_tag, audio_value in music_metadata.items():
                if audio_tag in ('title', 'track'):
                    continue
                # capitalize and remove any spaces
                audio_value = audio_value.title().replace(' ', '')
                # check that the tag is valid
                if valid_hash_tag(audio_value) and \
                   '#' + audio_value not in content:
                    # check that it hasn't been blocked
                    if not is_blocked_hashtag(base_dir, audio_value):
                        content += ' #' + audio_value

    tags = []
    hashtags_dict = {}

    domain = get_full_domain(domain, port)

    # add tags
    if nickname != 'news':
        content = \
            add_html_tags(base_dir, http_prefix,
                          nickname, domain, content,
                          mentioned_recipients,
                          hashtags_dict, translate, True)

    # replace emoji with unicode
    tags = []
    for tag_name, tag in hashtags_dict.items():
        tags.append(tag)

    # get list of tags
    if nickname != 'news':
        content = \
            replace_emoji_from_tags(None, base_dir, content, tags, 'content',
                                    False, True)
    # remove replaced emoji
    hashtags_dict_copy = hashtags_dict.copy()
    for tag_name, tag in hashtags_dict_copy.items():
        if tag.get('name'):
            if tag['name'].startswith(':'):
                if tag['name'] not in content:
                    del hashtags_dict[tag_name]

    status_number, published = get_status_number()
    new_post_id = \
        local_actor_url(http_prefix, nickname, domain) + \
        '/statuses/' + status_number

    sensitive = False
    summary = None
    if subject:
        summary = remove_invalid_chars(valid_content_warning(subject))
        sensitive = True

    to_recipients = []
    to_cc = []
    if to_url:
        if not isinstance(to_url, str):
            print('ERROR: to_url is not a string')
            return None
        to_recipients = [to_url]

    # who to send to
    if mentioned_recipients:
        for mention in mentioned_recipients:
            if mention not in to_cc:
                to_cc.append(mention)

    is_public = False
    for recipient in to_recipients:
        if recipient.endswith('#Public') or \
           recipient == 'as:Public' or \
           recipient == 'Public':
            is_public = True
            break

    # create a list of hashtags
    # Only posts which are #Public are searchable by hashtag
    if hashtags_dict:
        for tag_name, tag in hashtags_dict.items():
            if not post_tag_exists(tag['type'], tag['name'], tags):
                tags.append(tag)
            if is_public:
                _update_hashtags_index(base_dir, tag, new_post_id, nickname)
        # print('Content tags: ' + str(tags))

    sensitive, summary = \
        _create_post_cw_from_reply(base_dir, nickname, domain,
                                   in_reply_to, sensitive, summary,
                                   system_language, languages_understood)

    event_date_str = \
        _create_post_place_and_time(event_date, end_date,
                                    event_time, end_time,
                                    summary, content, schedule_post,
                                    event_uuid, location, tags)

    post_context = get_individual_post_context()

    if not is_public:
        # make sure that CC doesn't also contain a To address
        # eg. To: [ "https://mydomain/users/foo/followers" ]
        #     CC: [ "X", "Y", "https://mydomain/users/foo", "Z" ]
        remove_from_cc = []
        for cc_recipient in to_cc:
            for send_to_actor in to_recipients:
                if cc_recipient in send_to_actor and \
                   cc_recipient not in remove_from_cc:
                    remove_from_cc.append(cc_recipient)
                    break
        for cc_removal in remove_from_cc:
            to_cc.remove(cc_removal)
    else:
        if in_reply_to:
            # If this is a public post then get the actor being
            # replied to end ensure that it is within the CC list
            reply_actor = get_actor_from_in_reply_to(in_reply_to)
            if reply_actor:
                if reply_actor not in to_cc:
                    to_cc.append(reply_actor)

    # the type of post to be made
    post_object_type = 'Note'
    if is_article:
        post_object_type = 'Article'

    if not client_to_server:
        new_post = \
            _create_post_s2s(base_dir, nickname, domain, port,
                             http_prefix, content, status_number,
                             published, new_post_id, post_context,
                             to_recipients, to_cc, in_reply_to,
                             sensitive, comments_enabled,
                             tags, attach_image_filename,
                             media_type, image_description,
                             video_transcript, city,
                             post_object_type, summary,
                             in_reply_to_atom_uri, system_language,
                             conversation_id, low_bandwidth,
                             content_license_url, media_license_url,
                             media_creator, buy_url, chat_url,
                             translate)
    else:
        new_post = \
            _create_post_c2s(base_dir, nickname, domain, port,
                             http_prefix, content, status_number,
                             published, new_post_id, post_context,
                             to_recipients, to_cc, in_reply_to,
                             sensitive, comments_enabled,
                             tags, attach_image_filename,
                             media_type, image_description,
                             video_transcript, city,
                             post_object_type, summary,
                             in_reply_to_atom_uri, system_language,
                             conversation_id, low_bandwidth,
                             content_license_url, media_license_url,
                             media_creator, buy_url, chat_url,
                             translate)

    _create_post_mentions(cc_url, new_post, to_recipients, tags)

    _create_post_mod_report(base_dir, is_moderation_report,
                            new_post, new_post_id)

    # If a patch has been posted - i.e. the output from
    # git format-patch - then convert the activitypub type
    convert_post_to_patch(base_dir, nickname, domain, new_post)

    if schedule_post:
        if event_date and event_time:
            # add an item to the scheduled post index file
            _add_schedule_post(base_dir, nickname, domain,
                               event_date_str, new_post_id)
            save_post_to_box(base_dir, http_prefix, new_post_id,
                             nickname, domain, new_post, 'scheduled')
        else:
            print('Unable to create scheduled post without ' +
                  'date and time values')
            return new_post
    elif save_to_file:
        if is_article:
            save_post_to_box(base_dir, http_prefix, new_post_id,
                             nickname, domain, new_post, 'tlblogs')
        else:
            save_post_to_box(base_dir, http_prefix, new_post_id,
                             nickname, domain, new_post, 'outbox')
    return new_post


def outbox_message_create_wrap(http_prefix: str,
                               nickname: str, domain: str, port: int,
                               message_json: {}) -> {}:
    """Wraps a received message in a Create
    https://www.w3.org/TR/activitypub/#object-without-create
    """

    domain = get_full_domain(domain, port)
    status_number, published = get_status_number()
    if message_json.get('published'):
        published = message_json['published']
    new_post_id = \
        local_actor_url(http_prefix, nickname, domain) + \
        '/statuses/' + status_number
    cc_list = []
    if message_json.get('cc'):
        cc_list = message_json['cc']
    new_post = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'id': new_post_id + '/activity',
        'type': 'Create',
        'actor': local_actor_url(http_prefix, nickname, domain),
        'published': published,
        'to': message_json['to'],
        'cc': cc_list,
        'object': message_json
    }
    new_post['object']['id'] = new_post['id']
    new_post['object']['url'] = \
        http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
    new_post['object']['atomUri'] = \
        local_actor_url(http_prefix, nickname, domain) + \
        '/statuses/' + status_number
    return new_post


def _post_is_addressed_to_followers(nickname: str, domain: str, port: int,
                                    http_prefix: str,
                                    post_json_object: {}) -> bool:
    """Returns true if the given post is addressed to followers of the nickname
    """
    domain_full = get_full_domain(domain, port)

    if not post_json_object.get('object'):
        return False
    to_list = []
    cc_list = []
    if post_json_object['type'] != 'Update' and \
       has_object_dict(post_json_object):
        if post_json_object['object'].get('to'):
            if isinstance(post_json_object['object']['to'], list):
                to_list = post_json_object['object']['to']
            elif isinstance(post_json_object['object']['to'], str):
                to_list = [post_json_object['object']['to']]
        if post_json_object['object'].get('cc'):
            if isinstance(post_json_object['object']['cc'], list):
                cc_list = post_json_object['object']['cc']
            elif isinstance(post_json_object['object']['cc'], str):
                cc_list = [post_json_object['object']['cc']]
    else:
        if post_json_object.get('to'):
            if isinstance(post_json_object['to'], list):
                to_list = post_json_object['to']
            elif isinstance(post_json_object['to'], str):
                to_list = [post_json_object['to']]
        if post_json_object.get('cc'):
            if isinstance(post_json_object['cc'], list):
                cc_list = post_json_object['cc']
            elif isinstance(post_json_object['cc'], str):
                cc_list = [post_json_object['cc']]

    followers_url = \
        local_actor_url(http_prefix, nickname, domain_full) + '/followers'

    # does the followers url exist in 'to' or 'cc' lists?
    addressed_to_followers = False
    if followers_url in to_list:
        addressed_to_followers = True
    elif followers_url in cc_list:
        addressed_to_followers = True
    return addressed_to_followers


def pin_post2(base_dir: str, nickname: str, domain: str,
              pinned_content: str) -> None:
    """Pins the given post Id to the profile of then given account
    """
    account_dir = acct_dir(base_dir, nickname, domain)
    pinned_filename = account_dir + '/pinToProfile.txt'
    try:
        with open(pinned_filename, 'w+', encoding='utf-8') as pin_file:
            pin_file.write(pinned_content)
    except OSError:
        print('EX: unable to write ' + pinned_filename)


def undo_pinned_post(base_dir: str, nickname: str, domain: str) -> None:
    """Removes pinned content for then given account
    """
    account_dir = acct_dir(base_dir, nickname, domain)
    pinned_filename = account_dir + '/pinToProfile.txt'
    if not os.path.isfile(pinned_filename):
        return
    try:
        os.remove(pinned_filename)
    except OSError:
        print('EX: undo_pinned_post unable to delete ' + pinned_filename)


def get_pinned_post_as_json(base_dir: str, http_prefix: str,
                            nickname: str, domain: str,
                            domain_full: str, system_language: str) -> {}:
    """Returns the pinned profile post as json
    """
    account_dir = acct_dir(base_dir, nickname, domain)
    pinned_filename = account_dir + '/pinToProfile.txt'
    pinned_post_json = {}
    actor = local_actor_url(http_prefix, nickname, domain_full)
    if os.path.isfile(pinned_filename):
        pinned_content = None
        with open(pinned_filename, 'r', encoding='utf-8') as pin_file:
            pinned_content = pin_file.read()
        if pinned_content:
            pinned_post_json = {
                'atomUri': actor + '/pinned',
                'attachment': [],
                'attributedTo': actor,
                'cc': [
                    actor + '/followers'
                ],
                'content': pinned_content,
                'contentMap': {
                    system_language: pinned_content
                },
                'id': actor + '/pinned',
                'inReplyTo': None,
                'inReplyToAtomUri': None,
                'published': file_last_modified(pinned_filename),
                'replies': {},
                'sensitive': False,
                'summary': None,
                'tag': [],
                'to': ['https://www.w3.org/ns/activitystreams#Public'],
                'type': 'Note',
                'url': replace_users_with_at(actor) + '/pinned'
            }
    return pinned_post_json


def json_pin_post(base_dir: str, http_prefix: str,
                  nickname: str, domain: str,
                  domain_full: str, system_language: str) -> {}:
    """Returns a pinned post as json
    """
    pinned_post_json = \
        get_pinned_post_as_json(base_dir, http_prefix,
                                nickname, domain,
                                domain_full, system_language)
    items_list = []
    if pinned_post_json:
        items_list = [pinned_post_json]

    actor = local_actor_url(http_prefix, nickname, domain_full)
    post_context = get_individual_post_context()
    return {
        '@context': post_context,
        'id': actor + '/collections/featured',
        'orderedItems': items_list,
        'totalItems': len(items_list),
        'type': 'OrderedCollection'
    }


def regenerate_index_for_box(base_dir: str,
                             nickname: str, domain: str,
                             box_name: str) -> None:
    """Generates an index for the given box if it doesn't exist
    Used by unit tests to artificially create an index
    """
    box_dir = acct_dir(base_dir, nickname, domain) + '/' + box_name
    box_index_filename = box_dir + '.index'

    if not os.path.isdir(box_dir):
        return
    if os.path.isfile(box_index_filename):
        return

    index_lines = []
    for _, _, files in os.walk(box_dir):
        for fname in files:
            if ':##' not in fname:
                continue
            index_lines.append(fname)
        break

    index_lines.sort(reverse=True)

    result = ''
    try:
        with open(box_index_filename, 'w+', encoding='utf-8') as fp_box:
            for line in index_lines:
                result += line + '\n'
                fp_box.write(line + '\n')
    except OSError:
        print('EX: unable to generate index for ' + box_name + ' ' + result)
    print('Index generated for ' + box_name + '\n' + result)


def create_public_post(base_dir: str,
                       nickname: str, domain: str, port: int, http_prefix: str,
                       content: str, save_to_file: bool,
                       client_to_server: bool, comments_enabled: bool,
                       attach_image_filename: str, media_type: str,
                       image_description: str, video_transcript: str,
                       city: str, in_reply_to: str,
                       in_reply_to_atom_uri: str, subject: str,
                       schedule_post: bool,
                       event_date: str, event_time: str, event_end_time: str,
                       location: str, is_article: bool, system_language: str,
                       conversation_id: str, low_bandwidth: bool,
                       content_license_url: str,
                       media_license_url: str, media_creator: str,
                       languages_understood: [], translate: {},
                       buy_url: str, chat_url: str,
                       auto_cw_cache: {}) -> {}:
    """Public post
    """
    domain_full = get_full_domain(domain, port)
    is_moderation_report = False
    event_uuid = None
    category = None
    join_mode = None
    end_date = event_date
    end_time = event_end_time
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = None
    ticket_url = None
    local_actor = local_actor_url(http_prefix, nickname, domain_full)
    return _create_post_base(base_dir, nickname, domain, port,
                             'https://www.w3.org/ns/activitystreams#Public',
                             local_actor + '/followers',
                             http_prefix, content, save_to_file,
                             client_to_server, comments_enabled,
                             attach_image_filename, media_type,
                             image_description, video_transcript, city,
                             is_moderation_report, is_article,
                             in_reply_to, in_reply_to_atom_uri, subject,
                             schedule_post, event_date, event_time, location,
                             event_uuid, category, join_mode,
                             end_date, end_time,
                             maximum_attendee_capacity,
                             replies_moderation_option,
                             anonymous_participation_enabled,
                             event_status, ticket_url, system_language,
                             conversation_id, low_bandwidth,
                             content_license_url,
                             media_license_url, media_creator,
                             languages_understood, translate, buy_url,
                             chat_url, auto_cw_cache)


def create_reading_post(base_dir: str,
                        nickname: str, domain: str, port: int,
                        http_prefix: str,
                        mentions_str: str, reading_update_type: str,
                        book_title: str, book_url: str, book_rating: float,
                        save_to_file: bool,
                        client_to_server: bool, comments_enabled: bool,
                        attach_image_filename: str, media_type: str,
                        image_description: str, video_transcript: str,
                        city: str, in_reply_to: str,
                        in_reply_to_atom_uri: str, subject: str,
                        schedule_post: bool,
                        event_date: str, event_time: str, event_end_time: str,
                        location: str, is_article: bool, system_language: str,
                        conversation_id: str, low_bandwidth: bool,
                        content_license_url: str,
                        media_license_url: str, media_creator: str,
                        languages_understood: [], translate: {},
                        buy_url: str, chat_url: str,
                        auto_cw_cache: {}) -> {}:
    """ reading status post
    """
    content = ''
    if mentions_str:
        if not mentions_str.endswith(' '):
            mentions_str += ' '
    if reading_update_type == 'readingupdatewant':
        content = mentions_str + translate['wants to read'] + \
            ' <a href="' + book_url + \
            '"><i>' + book_title + '</i></a>'
    elif reading_update_type == 'readingupdateread':
        content = mentions_str + translate['reading'] + \
            ' <a href="' + book_url + \
            '"><i>' + book_title + '</i></a>'
    elif reading_update_type == 'readingupdatefinished':
        content = mentions_str + translate['finished reading'] + \
            ' <a href="' + book_url + \
            '"><i>' + book_title + '</i></a>'
    elif reading_update_type == 'readingupdaterating' and book_rating > 0:
        content = translate['rated'] + ' <a href="' + book_url + \
            '"><i>' + book_title + '</i></a>'
    if not content:
        return None
    post_json_object = \
        create_public_post(base_dir,
                           nickname, domain, port, http_prefix,
                           content, save_to_file,
                           client_to_server, comments_enabled,
                           attach_image_filename, media_type,
                           image_description, video_transcript,
                           city, in_reply_to,
                           in_reply_to_atom_uri, subject,
                           schedule_post,
                           event_date, event_time, event_end_time,
                           location, is_article, system_language,
                           conversation_id, low_bandwidth,
                           content_license_url,
                           media_license_url, media_creator,
                           languages_understood, translate,
                           buy_url, chat_url, auto_cw_cache)
    if post_json_object:
        post_json_object['object']['tag'] = [{
            'href': book_url,
            'name': book_title,
            'type': 'Edition'
        }]
    return post_json_object


def _append_citations_to_blog_post(base_dir: str,
                                   nickname: str, domain: str,
                                   blog_json: {}) -> None:
    """Appends any citations to a new blog post
    """
    # append citations tags, stored in a file
    citations_filename = \
        acct_dir(base_dir, nickname, domain) + '/.citations.txt'
    if not os.path.isfile(citations_filename):
        return
    citations_separator = '#####'
    with open(citations_filename, 'r', encoding='utf-8') as fp_cit:
        citations = fp_cit.readlines()
        for line in citations:
            if citations_separator not in line:
                continue
            sections = line.strip().split(citations_separator)
            if len(sections) != 3:
                continue
            # date_str = sections[0]
            title = sections[1]
            link = sections[2]
            tag_json = {
                "type": "Article",
                "name": title,
                "url": link
            }
            blog_json['object']['tag'].append(tag_json)


def create_blog_post(base_dir: str,
                     nickname: str, domain: str, port: int, http_prefix: str,
                     content: str, save_to_file: bool,
                     client_to_server: bool, comments_enabled: bool,
                     attach_image_filename: str, media_type: str,
                     image_description: str, video_transcript: str,
                     city: str, in_reply_to: str, in_reply_to_atom_uri: str,
                     subject: str, schedule_post: bool,
                     event_date: str, event_time: str, event_end_time: str,
                     location: str, system_language: str,
                     conversation_id: str, low_bandwidth: bool,
                     content_license_url: str,
                     media_license_url: str, media_creator: str,
                     languages_understood: [], translate: {},
                     buy_url: str, chat_url: str) -> {}:
    auto_cw_cache = {}
    blog_json = \
        create_public_post(base_dir,
                           nickname, domain, port, http_prefix,
                           content, save_to_file,
                           client_to_server, comments_enabled,
                           attach_image_filename, media_type,
                           image_description, video_transcript, city,
                           in_reply_to, in_reply_to_atom_uri, subject,
                           schedule_post,
                           event_date, event_time, event_end_time, location,
                           True, system_language, conversation_id,
                           low_bandwidth, content_license_url,
                           media_license_url, media_creator,
                           languages_understood, translate, buy_url, chat_url,
                           auto_cw_cache)
    url_str = get_url_from_post(blog_json['object']['url'])
    obj_url = remove_html(url_str)
    if '/@/' not in obj_url:
        blog_json['object']['url'] = obj_url.replace('/@', '/users/')
    _append_citations_to_blog_post(base_dir, nickname, domain, blog_json)

    return blog_json


def create_news_post(base_dir: str,
                     domain: str, port: int, http_prefix: str,
                     content: str, save_to_file: bool,
                     attach_image_filename: str, media_type: str,
                     image_description: str, video_transcript: str, city: str,
                     subject: str, system_language: str,
                     conversation_id: str, low_bandwidth: bool,
                     content_license_url: str,
                     media_license_url: str, media_creator: str,
                     languages_understood: [], translate: {},
                     buy_url: str, chat_url: str) -> {}:
    auto_cw_cache = {}
    client_to_server = False
    in_reply_to = None
    in_reply_to_atom_uri = None
    schedule_post = False
    event_date = None
    event_time = None
    event_end_time = None
    location = None
    blog = \
        create_public_post(base_dir,
                           'news', domain, port, http_prefix,
                           content, save_to_file,
                           client_to_server, False,
                           attach_image_filename, media_type,
                           image_description, video_transcript, city,
                           in_reply_to, in_reply_to_atom_uri, subject,
                           schedule_post,
                           event_date, event_time, event_end_time, location,
                           True, system_language, conversation_id,
                           low_bandwidth, content_license_url,
                           media_license_url, media_creator,
                           languages_understood, translate, buy_url, chat_url,
                           auto_cw_cache)
    blog['object']['type'] = 'Article'
    return blog


def create_question_post(base_dir: str,
                         nickname: str, domain: str, port: int,
                         http_prefix: str,
                         content: str, q_options: [],
                         save_to_file: bool,
                         client_to_server: bool, comments_enabled: bool,
                         attach_image_filename: str, media_type: str,
                         image_description: str, video_transcript: str,
                         city: str, subject: str, duration_days: int,
                         system_language: str, low_bandwidth: bool,
                         content_license_url: str,
                         media_license_url: str, media_creator: str,
                         languages_understood: [], translate: {},
                         auto_cw_cache: {}) -> {}:
    """Question post with multiple choice options
    """
    domain_full = get_full_domain(domain, port)
    local_actor = local_actor_url(http_prefix, nickname, domain_full)
    buy_url = ''
    chat_url = ''
    is_moderation_report = False
    is_article = False
    in_reply_to = in_reply_to_atom_uri = None
    schedule_post = False
    event_date = event_time = location = event_uuid = category = None
    join_mode = end_date = end_time = None
    maximum_attendee_capacity = replies_moderation_option = None
    anonymous_participation_enabled = event_status = ticket_url = None
    conversation_id = None
    message_json = \
        _create_post_base(base_dir, nickname, domain, port,
                          'https://www.w3.org/ns/activitystreams#Public',
                          local_actor + '/followers',
                          http_prefix, content, save_to_file,
                          client_to_server, comments_enabled,
                          attach_image_filename, media_type,
                          image_description, video_transcript, city,
                          is_moderation_report, is_article,
                          in_reply_to, in_reply_to_atom_uri, subject,
                          schedule_post, event_date, event_time,
                          location, event_uuid, category,
                          join_mode, end_date, end_time,
                          maximum_attendee_capacity,
                          replies_moderation_option,
                          anonymous_participation_enabled, event_status,
                          ticket_url, system_language,
                          conversation_id, low_bandwidth, content_license_url,
                          media_license_url, media_creator,
                          languages_understood, translate, buy_url,
                          chat_url, auto_cw_cache)
    message_json['object']['type'] = 'Question'
    message_json['object']['oneOf'] = []
    message_json['object']['votersCount'] = 0
    curr_time = date_utcnow()
    days_since_epoch = \
        int((curr_time - date_epoch()).days + duration_days)
    end_time = date_epoch() + \
        datetime.timedelta(days_since_epoch)
    message_json['object']['endTime'] = end_time.strftime("%Y-%m-%dT%H:%M:%SZ")
    for question_option in q_options:
        message_json['object']['oneOf'].append({
            "type": "Note",
            "name": question_option,
            "replies": {
                "type": "Collection",
                "totalItems": 0
            }
        })
    return message_json


def create_unlisted_post(base_dir: str,
                         nickname: str, domain: str, port: int,
                         http_prefix: str,
                         content: str, save_to_file: bool,
                         client_to_server: bool, comments_enabled: bool,
                         attach_image_filename: str, media_type: str,
                         image_description: str, video_transcript: str,
                         city: str, in_reply_to: str,
                         in_reply_to_atom_uri: str,
                         subject: str, schedule_post: bool,
                         event_date: str, event_time: str, event_end_time: str,
                         location: str, system_language: str,
                         conversation_id: str, low_bandwidth: bool,
                         content_license_url: str,
                         media_license_url: str, media_creator: str,
                         languages_understood: [], translate: {},
                         buy_url: str, chat_url: str,
                         auto_cw_cache: {}) -> {}:
    """Unlisted post. This has the #Public and followers links inverted.
    """
    domain_full = get_full_domain(domain, port)
    local_actor = local_actor_url(http_prefix, nickname, domain_full)
    is_moderation_report = False
    is_article = False
    event_uuid = category = join_mode = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = None
    ticket_url = None
    return _create_post_base(base_dir, nickname, domain, port,
                             local_actor + '/followers',
                             'https://www.w3.org/ns/activitystreams#Public',
                             http_prefix, content, save_to_file,
                             client_to_server, comments_enabled,
                             attach_image_filename, media_type,
                             image_description, video_transcript, city,
                             is_moderation_report, is_article,
                             in_reply_to, in_reply_to_atom_uri, subject,
                             schedule_post, event_date,
                             event_time, location,
                             event_uuid, category, join_mode,
                             event_date, event_end_time,
                             maximum_attendee_capacity,
                             replies_moderation_option,
                             anonymous_participation_enabled,
                             event_status,
                             ticket_url, system_language,
                             conversation_id, low_bandwidth,
                             content_license_url,
                             media_license_url, media_creator,
                             languages_understood, translate,
                             buy_url, chat_url, auto_cw_cache)


def create_followers_only_post(base_dir: str,
                               nickname: str, domain: str, port: int,
                               http_prefix: str, content: str,
                               save_to_file: bool,
                               client_to_server: bool, comments_enabled: bool,
                               attach_image_filename: str, media_type: str,
                               image_description: str, video_transcript: str,
                               city: str, in_reply_to: str,
                               in_reply_to_atom_uri: str,
                               subject: str, schedule_post: bool,
                               event_date: str,
                               event_time: str,  event_end_time: str,
                               location: str, system_language: str,
                               conversation_id: str, low_bandwidth: bool,
                               content_license_url: str,
                               media_license_url: str, media_creator: str,
                               languages_understood: [],
                               translate: {}, buy_url: str,
                               chat_url: str,
                               auto_cw_cache: {}) -> {}:
    """Followers only post
    """
    domain_full = get_full_domain(domain, port)
    local_actor = local_actor_url(http_prefix, nickname, domain_full)
    is_moderation_report = False
    is_article = False
    event_uuid = category = join_mode = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = None
    ticket_url = None
    return _create_post_base(base_dir, nickname, domain, port,
                             local_actor + '/followers', None,
                             http_prefix, content, save_to_file,
                             client_to_server, comments_enabled,
                             attach_image_filename, media_type,
                             image_description, video_transcript, city,
                             is_moderation_report, is_article,
                             in_reply_to, in_reply_to_atom_uri, subject,
                             schedule_post, event_date, event_time, location,
                             event_uuid, category, join_mode,
                             event_date, event_end_time,
                             maximum_attendee_capacity,
                             replies_moderation_option,
                             anonymous_participation_enabled,
                             event_status, ticket_url, system_language,
                             conversation_id, low_bandwidth,
                             content_license_url,
                             media_license_url, media_creator,
                             languages_understood, translate,
                             buy_url, chat_url, auto_cw_cache)


def get_mentioned_people(base_dir: str, http_prefix: str,
                         content: str, domain: str, debug: bool) -> []:
    """Extracts a list of mentioned actors from the given message content
    """
    if '@' not in content:
        return None
    mentions = []
    words = content.split(' ')
    for wrd in words:
        if not wrd.startswith('@'):
            continue
        handle = wrd[1:]
        if debug:
            print('DEBUG: mentioned handle ' + handle)
        if '@' not in handle:
            handle = handle + '@' + domain
            handle_dir = acct_handle_dir(base_dir, handle)
            if not os.path.isdir(handle_dir):
                continue
        else:
            external_domain = handle.split('@')[1]
            if not ('.' in external_domain or
                    external_domain == 'localhost'):
                continue
        mentioned_nickname = handle.split('@')[0]
        mentioned_domain = handle.split('@')[1].strip('\n').strip('\r')
        if ':' in mentioned_domain:
            mentioned_domain = remove_domain_port(mentioned_domain)
        if not valid_nickname(mentioned_domain, mentioned_nickname):
            continue
        actor = \
            local_actor_url(http_prefix, mentioned_nickname,
                            handle.split('@')[1])
        mentions.append(actor)
    return mentions


def create_direct_message_post(base_dir: str,
                               nickname: str, domain: str, port: int,
                               http_prefix: str, content: str,
                               save_to_file: bool, client_to_server: bool,
                               comments_enabled: bool,
                               attach_image_filename: str, media_type: str,
                               image_description: str, video_transcript: str,
                               city: str, in_reply_to: str,
                               in_reply_to_atom_uri: str,
                               subject: str, debug: bool,
                               schedule_post: bool,
                               event_date: str, event_time: str,
                               event_end_time: str,
                               location: str, system_language: str,
                               conversation_id: str, low_bandwidth: bool,
                               content_license_url: str,
                               media_license_url: str, media_creator: str,
                               languages_understood: [],
                               dm_is_chat: bool, translate: {},
                               buy_url: str, chat_url: str,
                               auto_cw_cache: {}) -> {}:
    """Direct Message post
    """
    content = resolve_petnames(base_dir, nickname, domain, content)
    mentioned_people = \
        get_mentioned_people(base_dir, http_prefix, content, domain, debug)
    if debug:
        print('mentioned_people: ' + str(mentioned_people))
    if not mentioned_people:
        return None
    post_to = None
    post_cc = None
    is_moderation_report = False
    is_article = False
    event_uuid = category = join_mode = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = None
    ticket_url = None
    message_json = \
        _create_post_base(base_dir, nickname, domain, port,
                          post_to, post_cc,
                          http_prefix, content, save_to_file,
                          client_to_server, comments_enabled,
                          attach_image_filename, media_type,
                          image_description, video_transcript, city,
                          is_moderation_report, is_article,
                          in_reply_to, in_reply_to_atom_uri, subject,
                          schedule_post, event_date, event_time, location,
                          event_uuid, category, join_mode,
                          event_date, event_end_time,
                          maximum_attendee_capacity,
                          replies_moderation_option,
                          anonymous_participation_enabled,
                          event_status, ticket_url, system_language,
                          conversation_id, low_bandwidth,
                          content_license_url,
                          media_license_url, media_creator,
                          languages_understood, translate, buy_url, chat_url,
                          auto_cw_cache)
    # mentioned recipients go into To rather than Cc
    message_json['to'] = message_json['object']['cc']
    if not isinstance(message_json['to'], list):
        message_json['to'] = [message_json['to']]
    message_json['object']['to'] = message_json['to']
    message_json['cc'] = []
    message_json['object']['cc'] = []
    if dm_is_chat:
        message_json['object']['type'] = 'ChatMessage'
    if schedule_post:
        post_id = remove_id_ending(message_json['object']['id'])
        save_post_to_box(base_dir, http_prefix, post_id,
                         nickname, domain, message_json, 'scheduled')
    return message_json


def create_report_post(base_dir: str,
                       nickname: str, domain: str, port: int, http_prefix: str,
                       content: str, save_to_file: bool,
                       client_to_server: bool, comments_enabled: bool,
                       attach_image_filename: str, media_type: str,
                       image_description: str, video_transcript: str,
                       city: str, debug: bool, subject: str,
                       system_language: str, low_bandwidth: bool,
                       content_license_url: str,
                       media_license_url: str, media_creator: str,
                       languages_understood: [], translate: {},
                       auto_cw_cache: {}) -> {}:
    """Send a report to moderators
    """
    domain_full = get_full_domain(domain, port)

    # add a title to distinguish moderation reports from other posts
    report_title = 'Moderation Report'
    if not subject:
        subject = report_title
    else:
        if not subject.startswith(report_title):
            subject = report_title + ': ' + subject

    # create the list of moderators from the moderators file
    moderators_list = []
    moderators_file = base_dir + '/accounts/moderators.txt'
    if os.path.isfile(moderators_file):
        with open(moderators_file, 'r', encoding='utf-8') as fp_mod:
            for line in fp_mod:
                line = line.strip('\n').strip('\r')
                if line.startswith('#'):
                    continue
                if line.startswith('/users/'):
                    line = line.replace('users', '')
                if line.startswith('@'):
                    line = line[1:]
                if '@' in line:
                    nick = line.split('@')[0]
                    moderator_actor = \
                        local_actor_url(http_prefix, nick, domain_full)
                    if moderator_actor not in moderators_list:
                        moderators_list.append(moderator_actor)
                    continue
                if line.startswith('http') or \
                   line.startswith('ipfs') or \
                   line.startswith('ipns') or \
                   line.startswith('hyper'):
                    # must be a local address - no remote moderators
                    if '://' + domain_full + '/' in line:
                        if line not in moderators_list:
                            moderators_list.append(line)
                else:
                    if '/' not in line:
                        moderator_actor = \
                            local_actor_url(http_prefix, line, domain_full)
                        if moderator_actor not in moderators_list:
                            moderators_list.append(moderator_actor)
    if len(moderators_list) == 0:
        # if there are no moderators then the admin becomes the moderator
        admin_nickname = get_config_param(base_dir, 'admin')
        if admin_nickname:
            local_actor = \
                local_actor_url(http_prefix, admin_nickname, domain_full)
            moderators_list.append(local_actor)
    if not moderators_list:
        return None
    if debug:
        print('DEBUG: Sending report to moderators')
        print(str(moderators_list))
    post_to = moderators_list
    post_cc = None
    post_json_object = None
    buy_url = ''
    chat_url = ''
    is_moderation_report = True
    is_article = False
    in_reply_to = in_reply_to_atom_uri = None
    schedule_post = False
    event_date = None
    event_time = None
    location = None
    event_uuid = None
    category = None
    join_mode = None
    end_date = None
    end_time = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = None
    ticket_url = None
    conversation_id = None
    for to_url in post_to:
        # who is this report going to?
        to_nickname = to_url.split('/users/')[1]
        handle = to_nickname + '@' + domain
        post_json_object = \
            _create_post_base(base_dir, nickname, domain, port,
                              to_url, post_cc,
                              http_prefix, content, save_to_file,
                              client_to_server, comments_enabled,
                              attach_image_filename, media_type,
                              image_description, video_transcript, city,
                              is_moderation_report, is_article,
                              in_reply_to, in_reply_to_atom_uri, subject,
                              schedule_post, event_date, event_time,
                              location, event_uuid, category,
                              join_mode, end_date, end_time,
                              maximum_attendee_capacity,
                              replies_moderation_option,
                              anonymous_participation_enabled,
                              event_status, ticket_url, system_language,
                              conversation_id, low_bandwidth,
                              content_license_url,
                              media_license_url, media_creator,
                              languages_understood, translate,
                              buy_url, chat_url, auto_cw_cache)
        if not post_json_object:
            continue

        # save a notification file so that the moderator
        # knows something new has appeared
        new_report_file = acct_handle_dir(base_dir, handle) + '/.newReport'
        if os.path.isfile(new_report_file):
            continue
        try:
            with open(new_report_file, 'w+', encoding='utf-8') as fp_report:
                fp_report.write(to_url + '/moderation')
        except OSError:
            print('EX: create_report_post unable to write ' + new_report_file)

    return post_json_object


def _add_send_block(base_dir: str, nickname: str, domain: str,
                    inbox_url: str) -> None:
    """Adds an inbox which is blocking this instance
    """
    send_block_filename = \
        acct_dir(base_dir, nickname, domain) + '/send_blocks.txt'

    inbox_url = inbox_url + '\n'
    if inbox_url.endswith('/inbox\n'):
        inbox_url = inbox_url.replace('/inbox\n', '\n')

    if not os.path.isfile(send_block_filename):
        try:
            with open(send_block_filename, 'w+',
                      encoding='utf-8') as fp_blocks:
                fp_blocks.write(inbox_url)
        except OSError:
            print('EX: _add_send_block unable to create ' +
                  send_block_filename)
        return

    if not text_in_file(inbox_url, send_block_filename, False):
        try:
            with open(send_block_filename, 'a+',
                      encoding='utf-8') as fp_blocks:
                fp_blocks.write(inbox_url)
        except OSError:
            print('EX: _add_send_block unable to write ' +
                  send_block_filename)


def _remove_send_block(base_dir: str, nickname: str, domain: str,
                       inbox_url: str) -> None:
    """Removes an inbox which is blocking this instance
    """
    send_block_filename = \
        acct_dir(base_dir, nickname, domain) + '/send_blocks.txt'

    inbox_url = inbox_url + '\n'
    if inbox_url.endswith('/inbox\n'):
        inbox_url = inbox_url.replace('/inbox\n', '\n')

    if not os.path.isfile(send_block_filename):
        return

    if text_in_file(inbox_url, send_block_filename, False):
        send_blocks_str = ''
        try:
            with open(send_block_filename, 'r',
                      encoding='utf-8') as fp_blocks:
                send_blocks_str = fp_blocks.read()
        except OSError:
            print('EX: _remove_send_block unable to read ' +
                  send_block_filename)
        send_blocks_str = send_blocks_str.replace(inbox_url, '')
        try:
            with open(send_block_filename, 'w+',
                      encoding='utf-8') as fp_blocks:
                fp_blocks.write(send_blocks_str)
        except OSError:
            print('EX: _remove_send_block unable to write ' +
                  send_block_filename)


def thread_send_post(session, post_json_str: str, federation_list: [],
                     inbox_url: str, base_dir: str,
                     signature_header_json: {},
                     signature_header_json_ld: {},
                     post_log: [], debug: bool,
                     http_prefix: str, domain_full: str,
                     nickname: str, domain: str) -> None:
    """Sends a with retries
    """
    tries = 0
    send_interval_sec = 30
    for _ in range(20):
        post_result = None
        unauthorized = False
        if debug:
            print('Getting post_json_string for ' + inbox_url)
        try:
            post_result, unauthorized, return_code = \
                post_json_string(session, post_json_str, federation_list,
                                 inbox_url, signature_header_json,
                                 debug, http_prefix, domain_full)
            if return_code in range(500, 600):
                # if an instance is returning a code which indicates that
                # it might have a runtime error, like 503, then don't
                # continue to post to it
                break
            if debug:
                print('Obtained post_json_string for ' + inbox_url +
                      ' unauthorized: ' + str(unauthorized))
        except BaseException as ex:
            print('ERROR: post_json_string failed ' + str(ex))

        if unauthorized:
            # try again with application/ld+json header
            post_result = None
            unauthorized = False
            if debug:
                print('Getting ld post_json_string for ' + inbox_url)
            try:
                post_result, unauthorized, return_code = \
                    post_json_string(session, post_json_str, federation_list,
                                     inbox_url, signature_header_json_ld,
                                     debug, http_prefix, domain_full)
                if return_code in range(500, 600):
                    # if an instance is returning a code which indicates that
                    # it might have a runtime error, like 503, then don't
                    # continue to post to it
                    break
                if debug:
                    print('Obtained ld post_json_string for ' + inbox_url +
                          ' unauthorized: ' + str(unauthorized))
            except BaseException as ex:
                print('ERROR: ld post_json_string failed ' + str(ex))

        if unauthorized:
            _add_send_block(base_dir, nickname, domain, inbox_url)
            if debug:
                print('WARN: thread_send_post: Post is unauthorized ' +
                      inbox_url + ' ' + post_json_str)
            else:
                print('WARN: thread_send_post: Post is unauthorized ' +
                      inbox_url)
            break
        if post_result:
            log_str = 'Success on try ' + str(tries) + ': ' + post_json_str
        else:
            log_str = 'Retry ' + str(tries) + ': ' + post_json_str
        post_log.append(log_str)
        # keep the length of the log finite
        # Don't accumulate massive files on systems with limited resources
        while len(post_log) > 16:
            post_log.pop(0)
        if debug:
            # save the log file
            post_log_filename = base_dir + '/post.log'
            if os.path.isfile(post_log_filename):
                try:
                    with open(post_log_filename, 'a+',
                              encoding='utf-8') as log_file:
                        log_file.write(log_str + '\n')
                except OSError:
                    print('EX: thread_send_post unable to append ' +
                          post_log_filename)
            else:
                try:
                    with open(post_log_filename, 'w+',
                              encoding='utf-8') as log_file:
                        log_file.write(log_str + '\n')
                except OSError:
                    print('EX: thread_send_post unable to write ' +
                          post_log_filename)

        if post_result:
            _remove_send_block(base_dir, nickname, domain, inbox_url)
            if debug:
                print('DEBUG: successful json post to ' + inbox_url)
            # our work here is done
            break
        if debug:
            print(post_json_str)
            print('DEBUG: json post to ' + inbox_url +
                  ' failed. Waiting for ' +
                  str(send_interval_sec) + ' seconds.')
        time.sleep(send_interval_sec)
        tries += 1


def _get_destination_inbox_domain(inbox_url: str, to_domain: str) -> str:
    """ the inbox domain being sent to isn't always the same
    as the destination instance domain. eg public-api.wordpress.com
    rather than the wordpress instance domain
    """
    inbox_url_domain, inbox_url_port = get_domain_from_actor(inbox_url)
    if not inbox_url_domain:
        return inbox_url
    inbox_domain = get_full_domain(inbox_url_domain, inbox_url_port)
    if inbox_domain != to_domain:
        print('_get_destination_inbox_domain ' +
              'inbox domain is not the same as instance domain ' +
              inbox_domain + ' != ' + to_domain)
        return inbox_domain
    return to_domain


def send_post(signing_priv_key_pem: str, project_version: str,
              session, base_dir: str, nickname: str, domain: str, port: int,
              to_nickname: str, to_domain: str, to_port: int, cc_str: str,
              http_prefix: str, content: str,
              save_to_file: bool, client_to_server: bool,
              comments_enabled: bool,
              attach_image_filename: str, media_type: str,
              image_description: str, video_transcript: str, city: str,
              federation_list: [], send_threads: [], post_log: [],
              cached_webfingers: {}, person_cache: {},
              is_article: bool, system_language: str,
              languages_understood: [],
              shared_items_federated_domains: [],
              shared_item_federation_tokens: {},
              low_bandwidth: bool, content_license_url: str,
              media_license_url: str, media_creator: str,
              translate: {}, buy_url: str, chat_url: str,
              auto_cw_cache: {},
              debug: bool, in_reply_to: str,
              in_reply_to_atom_uri: str, subject: str) -> int:
    """Post to another inbox. Used by unit tests.
    """
    with_digest = True
    conversation_id = None

    if to_nickname == 'inbox':
        # shared inbox actor on @domain@domain
        to_nickname = to_domain

    to_domain = get_full_domain(to_domain, to_port)

    handle = http_prefix + '://' + to_domain + '/@' + to_nickname

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  domain, project_version, debug, False,
                                  signing_priv_key_pem)
    if not wf_request:
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
              str(wf_request))
        return 1

    if not client_to_server:
        post_to_box = 'inbox'
    else:
        post_to_box = 'outbox'
        if is_article:
            post_to_box = 'tlblogs'

    # get the actor inbox for the To handle
    origin_domain = domain
    (inbox_url, _, pub_key, to_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            nickname, domain, post_to_box,
                            72533, system_language)

    if not inbox_url:
        return 3
    if not pub_key:
        return 4
    if not to_person_id:
        return 5
    # shared_inbox is optional

    is_moderation_report = False
    schedule_post = False
    event_date = event_time = location = None
    event_uuid = category = None
    join_mode = None
    end_date = None
    end_time = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = ticket_url = None
    post_json_object = \
        _create_post_base(base_dir, nickname, domain, port,
                          to_person_id, cc_str, http_prefix, content,
                          save_to_file, client_to_server,
                          comments_enabled,
                          attach_image_filename, media_type,
                          image_description, video_transcript, city,
                          is_moderation_report, is_article, in_reply_to,
                          in_reply_to_atom_uri, subject,
                          schedule_post,
                          event_date, event_time, location,
                          event_uuid, category,
                          join_mode,
                          end_date, end_time,
                          maximum_attendee_capacity,
                          replies_moderation_option,
                          anonymous_participation_enabled,
                          event_status, ticket_url, system_language,
                          conversation_id, low_bandwidth,
                          content_license_url,
                          media_license_url, media_creator,
                          languages_understood,
                          translate, buy_url, chat_url,
                          auto_cw_cache)

    # get the senders private key
    private_key_pem = get_person_key(nickname, domain, base_dir,
                                     'private', debug)
    if len(private_key_pem) == 0:
        return 6

    # NOTE: the inbox domain being sent to isn't always the same
    # as the destination instance domain. eg public-api.wordpress.com
    # rather than the wordpress instance domain
    to_domain = _get_destination_inbox_domain(inbox_url, to_domain)

    if to_domain not in inbox_url:
        return 7

    # remove the domain to leave the path on its own
    post_path = inbox_url.split(to_domain, 1)[1]

    if not post_json_object.get('signature'):
        try:
            signed_post_json_object = post_json_object.copy()
            generate_json_signature(signed_post_json_object, private_key_pem)
            post_json_object = signed_post_json_object
        except BaseException as ex:
            print('WARN: send_post failed to JSON-LD sign post, ' + str(ex))
            pprint(signed_post_json_object)

    # convert json to string so that there are no
    # subsequent conversions after creating message body digest
    post_json_str = json.dumps(post_json_object)

    # construct the http header, including the message body digest
    signature_header_json = \
        create_signed_header(None, private_key_pem, nickname, domain, port,
                             to_domain, to_port,
                             post_path, http_prefix, with_digest,
                             post_json_str, 'application/activity+json')
    signature_header_json_ld = \
        create_signed_header(None, private_key_pem, nickname, domain, port,
                             to_domain, to_port,
                             post_path, http_prefix, with_digest,
                             post_json_str, 'application/ld+json')

    # if the "to" domain is within the shared items
    # federation list then send the token for this domain
    # so that it can request a catalog
    domain_full = get_full_domain(domain, port)
    if to_domain in shared_items_federated_domains:
        if shared_item_federation_tokens.get(domain_full):
            signature_header_json['Origin'] = domain_full
            signature_header_json_ld['Origin'] = domain_full
            signature_header_json['SharesCatalog'] = \
                shared_item_federation_tokens[domain_full]
            signature_header_json_ld['SharesCatalog'] = \
                shared_item_federation_tokens[domain_full]
            if debug:
                print('SharesCatalog added to header')
        elif debug:
            print(domain_full + ' not in shared_item_federation_tokens')
    elif debug:
        print(to_domain + ' not in shared_items_federated_domains ' +
              str(shared_items_federated_domains))

    if debug:
        print('signature_header_json: ' + str(signature_header_json))

    # Keep the number of threads being used small
    while len(send_threads) > 1000:
        print('WARN: Maximum threads reached - killing send thread')
        send_threads[0].kill()
        send_threads.pop(0)
        print('WARN: thread killed')
    print('THREAD: thread_send_post')
    thr = \
        thread_with_trace(target=thread_send_post,
                          args=(session,
                                post_json_str,
                                federation_list,
                                inbox_url, base_dir,
                                signature_header_json.copy(),
                                signature_header_json_ld.copy(),
                                post_log, debug, http_prefix,
                                domain_full,
                                nickname, domain), daemon=True)
    send_threads.append(thr)
    begin_thread(thr, 'send_post')
    return 0


def send_post_via_server(signing_priv_key_pem: str, project_version: str,
                         base_dir: str, session,
                         from_nickname: str, password: str,
                         from_domain: str, from_port: int,
                         to_nickname: str, to_domain: str, to_port: int,
                         cc_str: str,
                         http_prefix: str, content: str,
                         comments_enabled: bool,
                         attach_image_filename: str, media_type: str,
                         image_description: str, video_transcript: str,
                         city: str, cached_webfingers: {}, person_cache: {},
                         is_article: bool, system_language: str,
                         languages_understood: [],
                         low_bandwidth: bool,
                         content_license_url: str,
                         media_license_url: str, media_creator: str,
                         event_date: str, event_time: str, event_end_time: str,
                         location: str, translate: {},
                         buy_url: str, chat_url: str, auto_cw_cache: {},
                         debug: bool, in_reply_to: str,
                         in_reply_to_atom_uri: str,
                         conversation_id: str, subject: str) -> int:
    """Send a post via a proxy (c2s)
    """
    if not session:
        print('WARN: No session for send_post_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, from_port)

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         from_domain_full, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: post webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: post webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'
    if is_article:
        post_to_box = 'tlblogs'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            from_nickname,
                            from_domain_full, post_to_box,
                            82796, system_language)
    if not inbox_url:
        if debug:
            print('DEBUG: post no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: post no actor was found for ' + handle)
        return 4

    # Get the json for the c2s post, not saving anything to file
    # Note that base_dir is set to None
    save_to_file = False
    client_to_server = True
    if to_domain.lower().endswith('public'):
        to_person_id = 'https://www.w3.org/ns/activitystreams#Public'
        cc_str = \
            local_actor_url(http_prefix, from_nickname, from_domain_full) + \
            '/followers'
    else:
        if to_domain.lower().endswith('followers') or \
           to_domain.lower().endswith('followersonly'):
            to_person_id = \
                local_actor_url(http_prefix,
                                from_nickname, from_domain_full) + \
                '/followers'
        else:
            to_domain_full = get_full_domain(to_domain, to_port)
            to_person_id = \
                local_actor_url(http_prefix, to_nickname, to_domain_full)

    is_moderation_report = False
    schedule_post = False
    event_uuid = category = join_mode = None
    maximum_attendee_capacity = None
    replies_moderation_option = None
    anonymous_participation_enabled = None
    event_status = ticket_url = None
    post_json_object = \
        _create_post_base(base_dir,
                          from_nickname, from_domain, from_port,
                          to_person_id, cc_str, http_prefix, content,
                          save_to_file, client_to_server,
                          comments_enabled,
                          attach_image_filename, media_type,
                          image_description, video_transcript, city,
                          is_moderation_report, is_article, in_reply_to,
                          in_reply_to_atom_uri, subject, schedule_post,
                          event_date, event_time, location,
                          event_uuid, category, join_mode,
                          event_date, event_end_time,
                          maximum_attendee_capacity,
                          replies_moderation_option,
                          anonymous_participation_enabled,
                          event_status, ticket_url, system_language,
                          conversation_id, low_bandwidth,
                          content_license_url,
                          media_license_url, media_creator,
                          languages_understood,
                          translate, buy_url, chat_url, auto_cw_cache)

    auth_header = create_basic_auth_header(from_nickname, password)

    if attach_image_filename:
        headers = {
            'host': from_domain_full,
            'Authorization': auth_header
        }
        post_result = \
            post_image(session, attach_image_filename, [],
                       inbox_url, headers, http_prefix, from_domain_full)
        if not post_result:
            if debug:
                print('DEBUG: post failed to upload image')
#            return 9

    headers = {
        'host': from_domain_full,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_dumps = json.dumps(post_json_object)
    post_result, unauthorized, return_code = \
        post_json_string(session, post_dumps, [],
                         inbox_url, headers, debug,
                         http_prefix, from_domain_full,
                         5, True)
    if not post_result:
        if debug:
            if unauthorized:
                print('DEBUG: POST failed for c2s to ' +
                      inbox_url + ' unathorized')
            else:
                print('DEBUG: POST failed for c2s to ' +
                      inbox_url + ' return code ' + str(return_code))
        return 5

    if debug:
        print('DEBUG: c2s POST success')
    return 0


def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}:
    """Returns a dictionary with followers grouped by domain
    """
    handle = nickname + '@' + domain
    followers_filename = acct_handle_dir(base_dir, handle) + '/followers.txt'
    if not os.path.isfile(followers_filename):
        return None
    grouped = {}
    with open(followers_filename, 'r', encoding='utf-8') as foll_file:
        for follower_handle in foll_file:
            if '@' not in follower_handle:
                continue
            fhandle1 = follower_handle.strip()
            fhandle = remove_eol(fhandle1)
            follower_domain = fhandle.split('@')[1]
            if not grouped.get(follower_domain):
                grouped[follower_domain] = [fhandle]
            else:
                grouped[follower_domain].append(fhandle)
    return grouped


def _add_followers_to_public_post(post_json_object: {}) -> None:
    """Adds followers entry to cc if it doesn't exist
    """
    if not post_json_object.get('actor'):
        return

    if isinstance(post_json_object['object'], str):
        if not post_json_object.get('to'):
            return
        if len(post_json_object['to']) > 1:
            return
        if len(post_json_object['to']) == 0:
            return
        if not post_json_object['to'][0].endswith('#Public'):
            if not post_json_object['to'][0] == 'as:Public':
                if not post_json_object['to'][0] == 'Public':
                    return
        if post_json_object.get('cc'):
            return
        actor_url = get_actor_from_post(post_json_object)
        post_json_object['cc'] = [actor_url + '/followers']
    elif has_object_dict(post_json_object):
        if not post_json_object['object'].get('to'):
            return
        if len(post_json_object['object']['to']) > 1:
            return
        if len(post_json_object['object']['to']) == 0:
            return
        if not post_json_object['object']['to'][0].endswith('#Public'):
            if not post_json_object['object']['to'][0] == 'as:Public':
                if not post_json_object['object']['to'][0] == 'Public':
                    return
        if post_json_object['object'].get('cc'):
            return
        actor_url = get_actor_from_post(post_json_object)
        post_json_object['object']['cc'] = \
            [actor_url + '/followers']


def send_signed_json(post_json_object: {}, session, base_dir: str,
                     nickname: str, domain: str, port: int,
                     to_nickname: str, to_domain: str,
                     to_port: int, http_prefix: str,
                     client_to_server: bool, federation_list: [],
                     send_threads: [], post_log: [], cached_webfingers: {},
                     person_cache: {}, debug: bool, project_version: str,
                     shared_items_token: str, group_account: bool,
                     signing_priv_key_pem: str,
                     source_id: int, curr_domain: str,
                     onion_domain: str, i2p_domain: str,
                     extra_headers: {}, sites_unavailable: [],
                     system_language: str) -> int:
    """Sends a signed json object to an inbox/outbox
    """
    if debug:
        print('DEBUG: send_signed_json start')
    if not session:
        print('WARN: No session specified for send_signed_json')
        return 8
    with_digest = True

    if to_domain.endswith('.onion') or to_domain.endswith('.i2p'):
        http_prefix = 'http'

    if to_nickname == 'inbox':
        # shared inbox actor on @domain@domain
        to_nickname = to_domain

    to_domain = get_full_domain(to_domain, to_port)

    to_domain_url = http_prefix + '://' + to_domain
    if not site_is_active(to_domain_url, 10, sites_unavailable):
        print('send_signed_json domain is inactive: ' + to_domain_url)
        return 9
    print('Domain is active: ' + to_domain_url)
    handle_base = to_domain_url + '/@'
    if to_nickname:
        handle = handle_base + to_nickname
    else:
        single_user_instance_nickname = 'dev'
        handle = handle_base + single_user_instance_nickname

    if debug:
        print('DEBUG: send_signed_json handle - ' + handle +
              ' to_port ' + str(to_port))

    # domain shown in the user agent
    ua_domain = curr_domain
    if to_domain.endswith('.onion'):
        ua_domain = onion_domain
    elif to_domain.endswith('.i2p'):
        ua_domain = i2p_domain

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  ua_domain, project_version, debug,
                                  group_account, signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: send_signed_json webfinger for ' +
                  handle + ' failed')
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: send_signed_json webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    if wf_request.get('errors'):
        if debug:
            print('DEBUG: send_signed_json webfinger for ' + handle +
                  ' failed with errors ' + str(wf_request['errors']))

    if not client_to_server:
        post_to_box = 'inbox'
    else:
        post_to_box = 'outbox'

    # get the actor inbox/outbox for the To handle
    origin_domain = domain
    (inbox_url, _, pub_key, to_person_id, shared_inbox_url, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            nickname, domain, post_to_box,
                            source_id, system_language)

    print("send_signed_json inbox_url: " + str(inbox_url))
    print("send_signed_json to_person_id: " + str(to_person_id))
    print("send_signed_json shared_inbox_url: " + str(shared_inbox_url))
    if inbox_url:
        if inbox_url.endswith('/actor/inbox') or \
           inbox_url.endswith('/instance.actor/inbox'):
            inbox_url = shared_inbox_url

    if not inbox_url:
        if debug:
            print('DEBUG: send_signed_json missing inbox_url')
        return 3

    if debug:
        print('DEBUG: send_signed_json sending to endpoint ' + inbox_url)

    if not pub_key:
        if debug:
            print('DEBUG: send_signed_json missing pubkey')
        return 4
    if not to_person_id:
        if debug:
            print('DEBUG: send_signed_json missing person_id')
        return 5
    # shared_inbox is optional

    # get the senders private key
    account_domain = origin_domain
    if onion_domain:
        if account_domain == onion_domain:
            account_domain = curr_domain
    if i2p_domain:
        if account_domain == i2p_domain:
            account_domain = curr_domain
    private_key_pem = \
        get_person_key(nickname, account_domain, base_dir, 'private', debug)
    if len(private_key_pem) == 0:
        if debug:
            print('DEBUG: send_signed_json private key not found for ' +
                  nickname + '@' + account_domain +
                  ' in ' + base_dir + '/keys/private')
        return 6

    # NOTE: the inbox domain being sent to isn't always the same
    # as the destination instance domain. eg public-api.wordpress.com
    # rather than the wordpress instance domain
    to_domain = _get_destination_inbox_domain(inbox_url, to_domain)

    if to_domain not in inbox_url:
        if debug:
            print('DEBUG: send_signed_json ' +
                  to_domain + ' is not in ' + inbox_url)
        return 7

    # remove the domain to leave the path on its own
    post_path = inbox_url.split(to_domain, 1)[1]

    _add_followers_to_public_post(post_json_object)

    if not post_json_object.get('signature'):
        try:
            signed_post_json_object = post_json_object.copy()
            generate_json_signature(signed_post_json_object, private_key_pem)
            post_json_object = signed_post_json_object
        except BaseException as ex:
            print('WARN: send_signed_json failed to JSON-LD sign post, ' +
                  str(ex))
            pprint(signed_post_json_object)

    # convert json to string so that there are no
    # subsequent conversions after creating message body digest
    post_json_str = json.dumps(post_json_object)

    # if the sender domain has changed from clearnet to onion or i2p
    # then change the content of the post accordingly
    if debug:
        print('send_signed_json checking for changed origin domain: ' +
              domain + ' ' + curr_domain)
    if domain != curr_domain:
        if not curr_domain.endswith('.onion') and \
           not curr_domain.endswith('.i2p'):
            if debug:
                print('send_signed_json ' +
                      'changing post content sender domain from ' +
                      curr_domain + ' to ' + domain)
            post_json_str = \
                post_json_str.replace(curr_domain, domain)

    # construct the http header, including the message body digest
    signature_header_json = \
        create_signed_header(None, private_key_pem, nickname, domain, port,
                             to_domain, to_port,
                             post_path, http_prefix, with_digest,
                             post_json_str,
                             'application/activity+json')
    signature_header_json_ld = \
        create_signed_header(None, private_key_pem, nickname, domain, port,
                             to_domain, to_port,
                             post_path, http_prefix, with_digest,
                             post_json_str,
                             'application/ld+json')
    # optionally add a token so that the receiving instance may access
    # your shared items catalog
    if shared_items_token:
        signature_header_json['Origin'] = get_full_domain(domain, port)
        signature_header_json['SharesCatalog'] = shared_items_token
    elif debug:
        print('send_signed_json not sending shared items federation token')

    # add any extra headers
    for header_title, header_text in extra_headers.items():
        signature_header_json[header_title] = header_text

    # Keep the number of threads being used small
    while len(send_threads) > 1000:
        print('WARN: send_signed_json maximum threads reached - ' +
              'killing send thread')
        send_threads[0].kill()
        send_threads.pop(0)
        print('WARN: send_signed_json thread killed')

    if debug:
        print('DEBUG: send_signed_json starting thread to send post')
        pprint(post_json_object)
    domain_full = get_full_domain(domain, port)
    print('THREAD: send_signed_json thread_send_post 2')
    thr = \
        thread_with_trace(target=thread_send_post,
                          args=(session,
                                post_json_str,
                                federation_list,
                                inbox_url, base_dir,
                                signature_header_json.copy(),
                                signature_header_json_ld.copy(),
                                post_log, debug,
                                http_prefix, domain_full,
                                nickname, domain), daemon=True)
    send_threads.append(thr)
    # begin_thread(thr, 'send_signed_json')
    return 0


def add_to_field(activity_type: str, post_json_object: {},
                 debug: bool) -> ({}, bool):
    """The Follow/Add/Remove activity doesn't have a 'to' field and so one
    needs to be added so that activity distribution happens in a consistent way
    Returns true if a 'to' field exists or was added
    """
    if post_json_object.get('to'):
        return post_json_object, True

    if debug:
        pprint(post_json_object)
        print('DEBUG: no "to" field when sending to named addresses 2')

    is_same_type = False
    to_field_added = False
    if post_json_object.get('object'):
        if isinstance(post_json_object['object'], str):
            if post_json_object.get('type'):
                if post_json_object['type'] == activity_type:
                    is_same_type = True
                    if debug:
                        print('DEBUG: ' +
                              'add_to_field1 "to" field assigned to ' +
                              activity_type)
                    to_address = post_json_object['object']
                    if '/statuses/' in to_address:
                        to_address = to_address.split('/statuses/')[0]
                    post_json_object['to'] = [to_address]
                    if debug:
                        print('DEBUG: "to" address added: ' + to_address)
                    to_field_added = True
        elif has_object_dict(post_json_object):
            # add a to field to bookmark add or remove
            if post_json_object.get('type') and \
               post_json_object.get('actor') and \
               post_json_object['object'].get('type'):
                if post_json_object['type'] == 'Add' or \
                   post_json_object['type'] == 'Remove':
                    if post_json_object['object']['type'] == 'Document':
                        actor_url = get_actor_from_post(post_json_object)
                        post_json_object['to'] = [actor_url]
                        post_json_object['object']['to'] = [actor_url]
                        to_field_added = True

            if not to_field_added and \
               post_json_object['object'].get('type'):
                if post_json_object['object']['type'] == activity_type:
                    is_same_type = True
                    if isinstance(post_json_object['object']['object'], str):
                        if debug:
                            print('DEBUG: add_to_field2 ' +
                                  '"to" field assigned to ' +
                                  activity_type)
                        to_address = post_json_object['object']['object']
                        if '/statuses/' in to_address:
                            to_address = to_address.split('/statuses/')[0]
                        post_json_object['object']['to'] = [to_address]
                        post_json_object['to'] = [to_address]
                        if debug:
                            print('DEBUG: "to" address added: ' + to_address)
                        to_field_added = True

    if not is_same_type:
        return post_json_object, True
    if to_field_added:
        return post_json_object, True
    return post_json_object, False


def _is_profile_update(post_json_object: {}) -> bool:
    """Is the given post a profile update?
    for actor updates there is no 'to' within the object
    """
    if post_json_object.get('type'):
        if has_object_string_type(post_json_object, False):
            if (post_json_object['type'] == 'Update' and
                (post_json_object['object']['type'] == 'Person' or
                 post_json_object['object']['type'] == 'Application' or
                 post_json_object['object']['type'] == 'Group' or
                 post_json_object['object']['type'] == 'Service')):
                return True
    return False


def _send_to_named_addresses(server, session, session_onion, session_i2p,
                             base_dir: str,
                             nickname: str, domain: str,
                             onion_domain: str, i2p_domain: str, port: int,
                             http_prefix: str, federation_list: [],
                             send_threads: [], post_log: [],
                             cached_webfingers: {}, person_cache: {},
                             post_json_object: {}, debug: bool,
                             project_version: str,
                             shared_items_federated_domains: [],
                             shared_item_federation_tokens: {},
                             signing_priv_key_pem: str,
                             proxy_type: str,
                             followers_sync_cache: {},
                             sites_unavailable: [],
                             system_language: str) -> None:
    """sends a post to the specific named addresses in to/cc
    """
    if not session:
        print('WARN: No session for sendToNamedAddresses')
        return
    if not post_json_object.get('object'):
        return
    is_profile_update = False
    if has_object_dict(post_json_object):
        if _is_profile_update(post_json_object):
            # use the original object, which has a 'to'
            recipients_object = post_json_object
            is_profile_update = True

        if not is_profile_update:
            if not post_json_object['object'].get('to'):
                if debug:
                    pprint(post_json_object)
                    print('DEBUG: ' +
                          'no "to" field when sending to named addresses')
                if has_object_string_type(post_json_object, debug):
                    if post_json_object['object']['type'] == 'Follow' or \
                       post_json_object['object']['type'] == 'Join':
                        post_json_obj2 = post_json_object['object']['object']
                        if isinstance(post_json_obj2, str):
                            if debug:
                                print('DEBUG: _send_to_named_addresses ' +
                                      '"to" field assigned to Follow')
                            post_json_object['object']['to'] = \
                                [post_json_object['object']['object']]
                if not post_json_object['object'].get('to'):
                    return
            recipients_object = post_json_object['object']
    else:
        post_json_object, field_added = \
            add_to_field('Follow', post_json_object, debug)
        if not field_added:
            return
        post_json_object, field_added = \
            add_to_field('Like', post_json_object, debug)
        if not field_added:
            return
        recipients_object = post_json_object

    recipients = []
    recipient_type = ('to', 'cc')
    for rtype in recipient_type:
        if not recipients_object.get(rtype):
            continue
        if isinstance(recipients_object[rtype], list):
            if debug:
                pprint(recipients_object)
                print('recipients_object: ' + str(recipients_object[rtype]))
            for address in recipients_object[rtype]:
                if not address:
                    continue
                if '/' not in address:
                    continue
                if address.endswith('#Public') or \
                   address == 'as:Public' or \
                   address == 'Public':
                    continue
                if address.endswith('/followers'):
                    continue
                recipients.append(address)
        elif isinstance(recipients_object[rtype], str):
            address = recipients_object[rtype]
            if address:
                if '/' in address:
                    if address.endswith('#Public') or \
                       address == 'as:Public' or \
                       address == 'Public':
                        continue
                    if address.endswith('/followers'):
                        continue
                    recipients.append(address)
    if not recipients:
        if debug:
            print('DEBUG: no individual recipients')
        return
    if debug:
        print('DEBUG: Sending individually addressed posts: ' +
              str(recipients))
    domain_full = get_full_domain(domain, port)
    # randomize the recipients list order, so that we are not favoring
    # any particular account in terms of delivery time
    random.shuffle(recipients)
    # this is after the message has arrived at the server
    client_to_server = False
    for address in recipients:
        to_nickname = get_nickname_from_actor(address)
        if not to_nickname:
            continue
        to_domain, to_port = get_domain_from_actor(address)
        if not to_domain:
            continue
        to_domain_full = get_full_domain(to_domain, to_port)
        # Don't send profile/actor updates to yourself
        if is_profile_update:
            if nickname == to_nickname and \
               domain_full == to_domain_full:
                if debug:
                    print('Not sending profile update to self. ' +
                          nickname + '@' + domain_full)
                continue

        # if we have an alt onion domain and we are sending to
        # another onion domain then switch the clearnet
        # domain for the onion one
        from_domain = domain
        from_domain_full = get_full_domain(domain, port)
        from_http_prefix = http_prefix
        curr_session = session
        curr_proxy_type = proxy_type
        session_type = 'default'
        if onion_domain:
            if not from_domain.endswith('.onion') and \
               to_domain.endswith('.onion'):
                from_domain = onion_domain
                from_domain_full = onion_domain
                from_http_prefix = 'http'
                curr_session = session_onion
                port = 80
                to_port = 80
                curr_proxy_type = 'tor'
                session_type = 'tor'
        if i2p_domain:
            if not from_domain.endswith('.i2p') and \
               to_domain.endswith('.i2p'):
                from_domain = i2p_domain
                from_domain_full = i2p_domain
                from_http_prefix = 'http'
                curr_session = session_i2p
                port = 80
                to_port = 80
                curr_proxy_type = 'i2p'
                session_type = 'i2p'

        extra_headers = {}
        # followers synchronization header
        # See https://github.com/mastodon/mastodon/pull/14510
        # https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-8fcf.md
        sending_actor = \
            from_http_prefix + '://' + from_domain_full + '/users/' + nickname
        _, followers_sync_hash = \
            update_followers_sync_cache(base_dir,
                                        nickname, domain,
                                        http_prefix, domain_full,
                                        to_domain_full,
                                        followers_sync_cache)
        if followers_sync_hash:
            collection_sync_str = \
                'collectionId="' + sending_actor + '/followers", ' + \
                'url="' + sending_actor + '/followers_synchronization", ' + \
                'digest="' + followers_sync_hash + '"'
            extra_headers["Collection-Synchronization"] = collection_sync_str
            if debug:
                print('DEBUG: extra_headers, ' + str(extra_headers))

        if debug:
            to_domain_full = get_full_domain(to_domain, to_port)
            print('DEBUG: Post sending s2s: ' +
                  nickname + '@' + from_domain_full +
                  ' to ' + to_nickname + '@' + to_domain_full)

        # if the "to" domain is within the shared items
        # federation list then send the token for this domain
        # so that it can request a catalog
        shared_items_token = None
        if to_domain in shared_items_federated_domains:
            if shared_item_federation_tokens.get(from_domain_full):
                shared_items_token = \
                    shared_item_federation_tokens[from_domain_full]

        group_account = has_group_type(base_dir, address, person_cache)

        if not curr_session:
            curr_session = create_session(curr_proxy_type)
            if server:
                if session_type == 'tor':
                    server.session_onion = curr_session
                elif session_type == 'i2p':
                    server.session_i2p = curr_session
                else:
                    server.session = curr_session

        send_signed_json(post_json_object, curr_session, base_dir,
                         nickname, from_domain, port,
                         to_nickname, to_domain, to_port,
                         from_http_prefix, client_to_server,
                         federation_list,
                         send_threads, post_log, cached_webfingers,
                         person_cache, debug, project_version,
                         shared_items_token, group_account,
                         signing_priv_key_pem, 34436782,
                         domain, onion_domain, i2p_domain,
                         extra_headers, sites_unavailable,
                         system_language)


def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
                                   base_dir: str, nickname: str, domain: str,
                                   onion_domain: str,
                                   i2p_domain: str, port: int,
                                   http_prefix: str, federation_list: [],
                                   send_threads: [], post_log: [],
                                   cached_webfingers: {}, person_cache: {},
                                   post_json_object: {}, debug: bool,
                                   project_version: str,
                                   shared_items_federated_domains: [],
                                   shared_item_federation_tokens: {},
                                   signing_priv_key_pem: str,
                                   proxy_type: str,
                                   followers_sync_cache: {},
                                   sites_unavailable: [],
                                   system_language: str):
    """Returns a thread used to send a post to named addresses
    """
    print('THREAD: _send_to_named_addresses')
    send_thread = \
        thread_with_trace(target=_send_to_named_addresses,
                          args=(server, session, session_onion, session_i2p,
                                base_dir, nickname, domain,
                                onion_domain, i2p_domain, port,
                                http_prefix, federation_list,
                                send_threads, post_log,
                                cached_webfingers, person_cache,
                                post_json_object, debug,
                                project_version,
                                shared_items_federated_domains,
                                shared_item_federation_tokens,
                                signing_priv_key_pem,
                                proxy_type,
                                followers_sync_cache,
                                sites_unavailable,
                                system_language), daemon=True)
    if not begin_thread(send_thread, 'send_to_named_addresses_thread'):
        print('WARN: socket error while starting ' +
              'thread to send to named addresses.')
        return None
    return send_thread


def _has_shared_inbox(session, http_prefix: str, domain: str,
                      debug: bool, signing_priv_key_pem: str,
                      ua_domain: str) -> bool:
    """Returns true if the given domain has a shared inbox
    This tries the new and the old way of webfingering the shared inbox
    """
    try_handles = []
    if ':' not in domain:
        try_handles.append(domain + '@' + domain)
    try_handles.append('inbox@' + domain)
    for handle in try_handles:
        wf_request = webfinger_handle(session, handle, http_prefix, {},
                                      ua_domain, __version__, debug, False,
                                      signing_priv_key_pem)
        if wf_request:
            if isinstance(wf_request, dict):
                if not wf_request.get('errors'):
                    return True
    return False


def _sending_profile_update(post_json_object: {}) -> bool:
    """Returns true if the given json is a profile update
    """
    if post_json_object['type'] != 'Update':
        return False
    if not has_object_string_type(post_json_object, False):
        return False
    activity_type = post_json_object['object']['type']
    if activity_type in ('Person', 'Application', 'Group', 'Service'):
        return True
    return False


def send_to_followers(server, session, session_onion, session_i2p,
                      base_dir: str, nickname: str, domain: str,
                      onion_domain: str, i2p_domain: str, port: int,
                      http_prefix: str, federation_list: [],
                      send_threads: [], post_log: [],
                      cached_webfingers: {}, person_cache: {},
                      post_json_object: {}, debug: bool,
                      project_version: str,
                      shared_items_federated_domains: [],
                      shared_item_federation_tokens: {},
                      signing_priv_key_pem: str,
                      sites_unavailable: [],
                      system_language: str) -> None:
    """sends a post to the followers of the given nickname
    """
    print('send_to_followers')
    if not _post_is_addressed_to_followers(nickname, domain,
                                           port, http_prefix,
                                           post_json_object):
        if debug:
            print('Post is not addressed to followers')
        return
    print('Post is addressed to followers')

    extra_headers = {}
    grouped = group_followers_by_domain(base_dir, nickname, domain)
    if not grouped:
        if debug:
            print('Post to followers did not resolve any domains')
        return
    print('Post to followers resolved domains')
    # print(str(grouped))

    # this is after the message has arrived at the server
    client_to_server = False

    curr_proxy_type = None
    if domain.endswith('.onion'):
        curr_proxy_type = 'tor'
    elif domain.endswith('.i2p'):
        curr_proxy_type = 'i2p'

    sending_start_time = date_utcnow()
    print('Sending post to followers begins ' +
          sending_start_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
    sending_ctr = 0

    # randomize the order of sending to instances
    randomized_instances = []
    for follower_domain, follower_handles in grouped.items():
        randomized_instances.append([follower_domain, follower_handles])
    random.shuffle(randomized_instances)

    # send out to each instance
    for group_send in randomized_instances:
        follower_domain = group_send[0]
        follower_handles = group_send[1]
        print('Sending post to followers progress ' +
              str(int(sending_ctr * 100 / len(grouped.items()))) + '% ' +
              follower_domain)
        sending_ctr += 1

        if debug:
            pprint(follower_handles)

        # if the followers domain is within the shared items
        # federation list then send the token for this domain
        # so that it can request a catalog
        shared_items_token = None
        if follower_domain in shared_items_federated_domains:
            domain_full = get_full_domain(domain, port)
            if shared_item_federation_tokens.get(domain_full):
                shared_items_token = shared_item_federation_tokens[domain_full]

        # check that the follower's domain is active
        follower_domain_url = http_prefix + '://' + follower_domain
        if not site_is_active(follower_domain_url, 10, sites_unavailable):
            print('Sending post to followers domain is inactive: ' +
                  follower_domain_url)
            continue
        print('Sending post to followers domain is active: ' +
              follower_domain_url)

        # select the appropriate session
        curr_session = session
        curr_http_prefix = http_prefix
        if onion_domain:
            if follower_domain.endswith('.onion'):
                curr_session = session_onion
                curr_http_prefix = 'http'
        if i2p_domain:
            if follower_domain.endswith('.i2p'):
                curr_session = session_i2p
                curr_http_prefix = 'http'

        # get the domain showin by the user agent
        ua_domain = domain
        if follower_domain.endswith('.onion'):
            ua_domain = onion_domain
        elif follower_domain.endswith('.i2p'):
            ua_domain = i2p_domain

        with_shared_inbox = \
            _has_shared_inbox(curr_session, curr_http_prefix, follower_domain,
                              debug, signing_priv_key_pem, ua_domain)
        if debug:
            if with_shared_inbox:
                print(follower_domain + ' has shared inbox')
        if not with_shared_inbox:
            print('Sending post to followers, ' + follower_domain +
                  ' does not have a shared inbox')

        to_port = port
        index = 0
        to_domain = follower_handles[index].split('@')[1]
        if ':' in to_domain:
            to_port = get_port_from_domain(to_domain)
            to_domain = remove_domain_port(to_domain)

        # if we are sending to an onion domain and we
        # have an alt onion domain then use the alt
        from_domain = domain
        from_http_prefix = http_prefix
        session_type = 'default'
        if onion_domain:
            if to_domain.endswith('.onion'):
                from_domain = onion_domain
                from_http_prefix = 'http'
                port = 80
                to_port = 80
                curr_proxy_type = 'tor'
                session_type = 'tor'
        if i2p_domain:
            if to_domain.endswith('.i2p'):
                from_domain = i2p_domain
                from_http_prefix = 'http'
                port = 80
                to_port = 80
                curr_proxy_type = 'i2p'
                session_type = 'i2p'

        if not curr_session:
            curr_session = create_session(curr_proxy_type)
            if server:
                if session_type == 'tor':
                    server.session_onion = curr_session
                elif session_type == 'i2p':
                    server.session_i2p = curr_session
                else:
                    server.session = curr_session

        if with_shared_inbox:
            to_nickname = follower_handles[index].split('@')[0]

            group_account = False
            if to_nickname.startswith('!'):
                group_account = True
                to_nickname = to_nickname[1:]

            # if there are more than one followers on the domain
            # then send the post to the shared inbox
            if len(follower_handles) > 1:
                to_nickname = 'inbox'

            if to_nickname != 'inbox' and post_json_object.get('type'):
                if _sending_profile_update(post_json_object):
                    print('Sending post to followers ' +
                          'shared inbox of ' + to_domain)
                    to_nickname = 'inbox'

            print('Sending post to followers from ' +
                  nickname + '@' + domain +
                  ' to ' + to_nickname + '@' + to_domain)

            send_signed_json(post_json_object, curr_session, base_dir,
                             nickname, from_domain, port,
                             to_nickname, to_domain, to_port,
                             from_http_prefix,
                             client_to_server, federation_list,
                             send_threads, post_log, cached_webfingers,
                             person_cache, debug, project_version,
                             shared_items_token, group_account,
                             signing_priv_key_pem, 639342,
                             domain, onion_domain, i2p_domain,
                             extra_headers, sites_unavailable,
                             system_language)
        else:
            # randomize the order of handles, so that we are not
            # favoring any particular account in terms of its delivery time
            random.shuffle(follower_handles)
            # send to individual followers without using a shared inbox
            for handle in follower_handles:
                print('Sending post to followers ' + handle)
                to_nickname = handle.split('@')[0]

                group_account = False
                if to_nickname.startswith('!'):
                    group_account = True
                    to_nickname = to_nickname[1:]

                if post_json_object['type'] != 'Update':
                    print('Sending post to followers from ' +
                          nickname + '@' + domain + ' to ' +
                          to_nickname + '@' + to_domain)
                else:
                    print('Sending post to followers profile update from ' +
                          nickname + '@' + domain + ' to ' +
                          to_nickname + '@' + to_domain)

                send_signed_json(post_json_object, curr_session, base_dir,
                                 nickname, from_domain, port,
                                 to_nickname, to_domain, to_port,
                                 from_http_prefix,
                                 client_to_server, federation_list,
                                 send_threads, post_log, cached_webfingers,
                                 person_cache, debug, project_version,
                                 shared_items_token, group_account,
                                 signing_priv_key_pem, 634219,
                                 domain, onion_domain, i2p_domain,
                                 extra_headers, sites_unavailable,
                                 system_language)

        time.sleep(4)

    if debug:
        print('DEBUG: End of send_to_followers')

    sending_end_time = date_utcnow()
    sending_mins = \
        int((sending_end_time - sending_start_time).total_seconds() / 60)
    print('Sending post to followers ends ' + str(sending_mins) + ' mins')


def send_to_followers_thread(server, session, session_onion, session_i2p,
                             base_dir: str, nickname: str, domain: str,
                             onion_domain: str, i2p_domain: str, port: int,
                             http_prefix: str, federation_list: [],
                             send_threads: [], post_log: [],
                             cached_webfingers: {}, person_cache: {},
                             post_json_object: {}, debug: bool,
                             project_version: str,
                             shared_items_federated_domains: [],
                             shared_item_federation_tokens: {},
                             signing_priv_key_pem: str,
                             sites_unavailable: [],
                             system_language: str):
    """Returns a thread used to send a post to followers
    """
    print('THREAD: send_to_followers')
    send_thread = \
        thread_with_trace(target=send_to_followers,
                          args=(server, session, session_onion, session_i2p,
                                base_dir, nickname, domain,
                                onion_domain, i2p_domain, port,
                                http_prefix, federation_list,
                                send_threads, post_log,
                                cached_webfingers, person_cache,
                                post_json_object.copy(), debug,
                                project_version,
                                shared_items_federated_domains,
                                shared_item_federation_tokens,
                                signing_priv_key_pem,
                                sites_unavailable,
                                system_language), daemon=True)
    if not begin_thread(send_thread, 'send_to_followers_thread'):
        print('WARN: error while starting ' +
              'thread to send to followers.')
        return None
    return send_thread


def create_inbox(recent_posts_cache: {},
                 base_dir: str, nickname: str, domain: str, port: int,
                 http_prefix: str, items_per_page: int, header_only: bool,
                 page_number: int, first_post_id: str) -> {}:
    return _create_box_indexed(recent_posts_cache,
                               base_dir, 'inbox',
                               nickname, domain, port, http_prefix,
                               items_per_page, header_only, True,
                               0, False, 0, page_number, first_post_id)


def create_bookmarks_timeline(base_dir: str,
                              nickname: str, domain: str,
                              port: int, http_prefix: str, items_per_page: int,
                              header_only: bool, page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'tlbookmarks',
                               nickname, domain,
                               port, http_prefix, items_per_page, header_only,
                               True, 0, False, 0, page_number, '')


def create_dm_timeline(recent_posts_cache: {},
                       base_dir: str, nickname: str, domain: str,
                       port: int, http_prefix: str, items_per_page: int,
                       header_only: bool, page_number: int,
                       first_post_id: str) -> {}:
    return _create_box_indexed(recent_posts_cache,
                               base_dir, 'dm', nickname,
                               domain, port, http_prefix, items_per_page,
                               header_only, True, 0, False, 0, page_number,
                               first_post_id)


def create_replies_timeline(recent_posts_cache: {},
                            base_dir: str, nickname: str, domain: str,
                            port: int, http_prefix: str, items_per_page: int,
                            header_only: bool, page_number: int,
                            first_post_id: str) -> {}:
    return _create_box_indexed(recent_posts_cache,
                               base_dir, 'tlreplies',
                               nickname, domain, port, http_prefix,
                               items_per_page, header_only, True,
                               0, False, 0, page_number, first_post_id)


def create_blogs_timeline(base_dir: str, nickname: str, domain: str,
                          port: int, http_prefix: str, items_per_page: int,
                          header_only: bool, page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'tlblogs', nickname,
                               domain, port, http_prefix,
                               items_per_page, header_only, True,
                               0, False, 0, page_number, '')


def create_features_timeline(base_dir: str,
                             nickname: str, domain: str,
                             port: int, http_prefix: str, items_per_page: int,
                             header_only: bool, page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'tlfeatures', nickname,
                               domain, port, http_prefix,
                               items_per_page, header_only, True,
                               0, False, 0, page_number, '')


def create_media_timeline(base_dir: str, nickname: str, domain: str,
                          port: int, http_prefix: str, items_per_page: int,
                          header_only: bool, page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'tlmedia', nickname,
                               domain, port, http_prefix,
                               items_per_page, header_only, True,
                               0, False, 0, page_number, '')


def create_news_timeline(base_dir: str, domain: str,
                         port: int, http_prefix: str, items_per_page: int,
                         header_only: bool, newswire_votes_threshold: int,
                         positive_voting: bool, voting_time_mins: int,
                         page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'outbox', 'news',
                               domain, port, http_prefix,
                               items_per_page, header_only, True,
                               newswire_votes_threshold, positive_voting,
                               voting_time_mins, page_number, '')


def create_outbox(base_dir: str, nickname: str, domain: str,
                  port: int, http_prefix: str,
                  items_per_page: int, header_only: bool, authorized: bool,
                  page_number: int) -> {}:
    return _create_box_indexed({}, base_dir, 'outbox',
                               nickname, domain, port, http_prefix,
                               items_per_page, header_only, authorized,
                               0, False, 0, page_number, '')


def create_moderation(base_dir: str, nickname: str, domain: str, port: int,
                      http_prefix: str, items_per_page: int, header_only: bool,
                      page_number: int) -> {}:
    box_dir = create_person_dir(nickname, domain, base_dir, 'inbox')
    boxname = 'moderation'

    domain = get_full_domain(domain, port)

    if not page_number:
        page_number = 1

    page_str = '?page=' + str(page_number)
    box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
    box_header = {
        '@context': 'https://www.w3.org/ns/activitystreams',
        'first': box_url + '?page=true',
        'id': box_url,
        'last': box_url + '?page=true',
        'totalItems': 0,
        'type': 'OrderedCollection'
    }
    box_items = {
        '@context': 'https://www.w3.org/ns/activitystreams',
        'id': box_url + page_str,
        'orderedItems': [
        ],
        'partOf': box_url,
        'type': 'OrderedCollectionPage'
    }

    if is_moderator(base_dir, nickname):
        moderation_index_file = base_dir + '/accounts/moderation.txt'
        if os.path.isfile(moderation_index_file):
            with open(moderation_index_file, 'r',
                      encoding='utf-8') as index_file:
                lines = index_file.readlines()
            box_header['totalItems'] = len(lines)
            if header_only:
                return box_header

            page_lines = []
            if len(lines) > 0:
                end_line_number = \
                    len(lines) - 1 - int(items_per_page * page_number)
                end_line_number = max(end_line_number, 0)
                start_line_number = \
                    len(lines) - 1 - int(items_per_page * (page_number - 1))
                start_line_number = max(start_line_number, 0)
                line_number = start_line_number
                while line_number >= end_line_number:
                    line_no_str = lines[line_number].strip('\n').strip('\r')
                    page_lines.append(line_no_str)
                    line_number -= 1

            for post_url in page_lines:
                post_filename = \
                    box_dir + '/' + post_url.replace('/', '#') + '.json'
                if os.path.isfile(post_filename):
                    post_json_object = load_json(post_filename)
                    if post_json_object:
                        box_items['orderedItems'].append(post_json_object)

    if header_only:
        return box_header
    return box_items


def is_image_media(session, base_dir: str, http_prefix: str,
                   nickname: str, domain: str,
                   post_json_object: {},
                   yt_replace_domain: str,
                   twitter_replacement_domain: str,
                   allow_local_network_access: bool,
                   recent_posts_cache: {}, debug: bool,
                   system_language: str,
                   domain_full: str, person_cache: {},
                   signing_priv_key_pem: str,
                   bold_reading: bool,
                   show_vote_posts: bool,
                   languages_understood: []) -> bool:
    """Returns true if the given post has attached image media
    """
    if post_json_object['type'] == 'Announce':
        blocked_cache = {}
        block_federated = []
        post_json_announce = \
            download_announce(session, base_dir, http_prefix,
                              nickname, domain, post_json_object,
                              __version__,
                              yt_replace_domain,
                              twitter_replacement_domain,
                              allow_local_network_access,
                              recent_posts_cache, debug,
                              system_language,
                              domain_full, person_cache,
                              signing_priv_key_pem,
                              blocked_cache, block_federated,
                              bold_reading,
                              show_vote_posts,
                              languages_understood)
        if post_json_announce:
            post_json_object = post_json_announce
    if post_json_object['type'] != 'Create':
        return False
    if not has_object_dict(post_json_object):
        return False
    if post_json_object['object'].get('moderationStatus'):
        return False
    if post_json_object['object']['type'] not in ('Note', 'Page', 'Event',
                                                  'ChatMessage', 'Article'):
        return False
    post_attachments = get_post_attachments(post_json_object)
    if not post_attachments:
        return False
    for attach in post_attachments:
        if attach.get('mediaType') and attach.get('url'):
            if attach['mediaType'].startswith('image/') or \
               attach['mediaType'].startswith('audio/') or \
               attach['mediaType'].startswith('video/'):
                return True
    return False


def _add_post_string_to_timeline(post_str: str, boxname: str,
                                 posts_in_box: [], box_actor: str) -> bool:
    """ is this a valid timeline post?
    """
    # must be a recognized ActivityPub type
    if (string_contains(post_str,
                        ('"Note"', '"EncryptedMessage"', '"ChatMessage"',
                         '"Event"', '"Article"', '"Patch"', '"Announce"')) or
        ('"Question"' in post_str and
         ('"Create"' in post_str or '"Update"' in post_str))):

        if boxname == 'dm':
            if '#Public' in post_str or \
               '/followers' in post_str:
                return False
        elif boxname == 'tlreplies':
            if box_actor not in post_str:
                return False
        elif boxname in ('tlblogs', 'tlnews', 'tlfeatures'):
            if '"Create"' not in post_str:
                return False
            if '"Article"' not in post_str:
                return False
        elif boxname == 'tlmedia':
            if '"Create"' in post_str:
                if ('mediaType' not in post_str or
                    ('image/' not in post_str and
                     'video/' not in post_str and
                     'audio/' not in post_str)):
                    return False
        # add the post to the dictionary
        posts_in_box.append(post_str)
        return True
    return False


def _add_post_to_timeline(file_path: str, boxname: str,
                          posts_in_box: [], box_actor: str) -> bool:
    """ Reads a post from file and decides whether it is valid
    """
    with open(file_path, 'r', encoding='utf-8') as post_file:
        post_str = post_file.read()

        if file_path.endswith('.json'):
            replies_filename = file_path.replace('.json', '.replies')
            if os.path.isfile(replies_filename):
                # append a replies identifier, which will later be removed
                post_str += '<hasReplies>'

            mitm_filename = file_path.replace('.json', '.mitm')
            if os.path.isfile(mitm_filename):
                # append a mitm identifier, which will later be removed
                post_str += '<postmitm>'

        return _add_post_string_to_timeline(post_str, boxname, posts_in_box,
                                            box_actor)
    return False


def remove_post_interactions(post_json_object: {}, force: bool) -> bool:
    """ Don't show likes, replies, bookmarks, DMs or shares (announces) to
    unauthorized viewers. This makes the timeline less useful to
    marketers and other surveillance-oriented organizations.
    Returns False if this is a private post
    """
    has_object = False
    if has_object_dict(post_json_object):
        has_object = True
    if has_object:
        post_obj = post_json_object['object']
        if not force:
            # If not authorized and it's a private post
            # then just don't show it within timelines
            if not is_public_post(post_json_object):
                return False
    else:
        post_obj = post_json_object

    # clear the likes
    if post_obj.get('likes'):
        post_obj['likes'] = {
            'items': []
        }
    # clear the reactions
    if post_obj.get('reactions'):
        post_obj['reactions'] = {
            'items': []
        }
    # remove other collections
    remove_collections = (
        'replies', 'shares', 'bookmarks', 'ignores'
    )
    for remove_name in remove_collections:
        if post_obj.get(remove_name):
            post_obj[remove_name] = {}
    return True


def _passed_newswire_voting(newswire_votes_threshold: int,
                            base_dir: str, domain: str,
                            post_filename: str,
                            positive_voting: bool,
                            voting_time_mins: int) -> bool:
    """Returns true if the post has passed through newswire voting
    """
    # apply votes within this timeline
    if newswire_votes_threshold <= 0:
        return True
    # note that the presence of an arrival file also indicates
    # that this post is moderated
    arrival_date = \
        locate_news_arrival(base_dir, domain, post_filename)
    if not arrival_date:
        return True
    # how long has elapsed since this post arrived?
    curr_date = date_utcnow()
    time_diff_mins = \
        int((curr_date - arrival_date).total_seconds() / 60)
    # has the voting time elapsed?
    if time_diff_mins < voting_time_mins:
        # voting is still happening, so don't add this
        # post to the timeline
        return False
    # if there a votes file for this post?
    votes_filename = \
        locate_news_votes(base_dir, domain, post_filename)
    if not votes_filename:
        return True
    # load the votes file and count the votes
    votes_json = load_json(votes_filename, 0, 2)
    if not votes_json:
        return True
    if not positive_voting:
        if votes_on_newswire_item(votes_json) >= \
           newswire_votes_threshold:
            # Too many veto votes.
            # Continue without incrementing
            # the posts counter
            return False
    else:
        if votes_on_newswire_item < \
           newswire_votes_threshold:
            # Not enough votes.
            # Continue without incrementing
            # the posts counter
            return False
    return True


def _create_box_items(base_dir: str,
                      timeline_nickname: str,
                      original_domain: str,
                      nickname: str, domain: str,
                      index_box_name: str,
                      first_post_id: str,
                      page_number: int,
                      items_per_page: int,
                      newswire_votes_threshold: int,
                      positive_voting: bool,
                      voting_time_mins: int,
                      post_urls_in_box: [],
                      recent_posts_cache: {},
                      boxname: str,
                      posts_in_box: [],
                      box_actor: str) -> (int, int):
    """Creates the list of posts within a timeline
    """
    index_filename = \
        acct_dir(base_dir, timeline_nickname, original_domain) + \
        '/' + index_box_name + '.index'
    total_posts_count = 0
    posts_added_to_timeline = 0
    if not os.path.isfile(index_filename):
        return total_posts_count, posts_added_to_timeline

    # format the first post into an hashed url
    # Why are url's hashed? Since storage is in the filesystem this avoids
    # confusion with directories by not using the / character
    if first_post_id:
        first_post_id = first_post_id.replace('--', '#')
        first_post_id = first_post_id.replace('/', '#')

    with open(index_filename, 'r', encoding='utf-8') as index_file:
        posts_added_to_timeline = 0
        while posts_added_to_timeline < items_per_page:
            post_filename = index_file.readline()

            if not post_filename:
                break

            # if a first post is specified then wait until it is found
            # before starting to generate the timeline
            if first_post_id and total_posts_count == 0:
                if first_post_id not in post_filename:
                    continue
                total_posts_count = \
                    int((page_number - 1) * items_per_page)

            # Has this post passed through the newswire voting stage?
            if not _passed_newswire_voting(newswire_votes_threshold,
                                           base_dir, domain,
                                           post_filename,
                                           positive_voting,
                                           voting_time_mins):
                continue

            # Skip through any posts previous to the current page
            if not first_post_id:
                if total_posts_count < \
                   int((page_number - 1) * items_per_page):
                    total_posts_count += 1
                    continue

            # if this is a full path then remove the directories
            if '/' in post_filename:
                post_filename = post_filename.split('/')[-1]

            # filename of the post without any extension or path
            # This should also correspond to any index entry in
            # the posts cache
            post_url = remove_eol(post_filename)
            post_url = post_url.replace('.json', '').strip()

            # is this a duplicate?
            if post_url in post_urls_in_box:
                continue

            # is the post cached in memory?
            if recent_posts_cache.get('index'):
                if post_url in recent_posts_cache['index']:
                    if recent_posts_cache['json'].get(post_url):
                        url = recent_posts_cache['json'][post_url]
                        if _add_post_string_to_timeline(url,
                                                        boxname,
                                                        posts_in_box,
                                                        box_actor):
                            total_posts_count += 1
                            posts_added_to_timeline += 1
                            post_urls_in_box.append(post_url)
                            continue
                        print('Post not added to timeline')

            # read the post from file
            full_post_filename = \
                locate_post(base_dir, nickname,
                            original_domain, post_url, False)
            if full_post_filename:
                # has the post been rejected?
                if os.path.isfile(full_post_filename + '.reject'):
                    continue

                if _add_post_to_timeline(full_post_filename, boxname,
                                         posts_in_box, box_actor):
                    posts_added_to_timeline += 1
                    total_posts_count += 1
                    post_urls_in_box.append(post_url)
                else:
                    print('WARN: Unable to add post ' + post_url +
                          ' nickname ' + nickname +
                          ' timeline ' + boxname)
            else:
                if timeline_nickname != nickname:
                    # if this is the features timeline
                    full_post_filename = \
                        locate_post(base_dir, timeline_nickname,
                                    original_domain, post_url, False)
                    if full_post_filename:
                        if _add_post_to_timeline(full_post_filename,
                                                 boxname,
                                                 posts_in_box, box_actor):
                            posts_added_to_timeline += 1
                            total_posts_count += 1
                            post_urls_in_box.append(post_url)
                        else:
                            print('WARN: Unable to add features post ' +
                                  post_url + ' nickname ' + nickname +
                                  ' timeline ' + boxname)
                    else:
                        print('WARN: features timeline. ' +
                              'Unable to locate post ' + post_url)
                else:
                    if timeline_nickname == 'news':
                        print('WARN: Unable to locate news post ' +
                              post_url + ' nickname ' + nickname)
                    else:
                        print('WARN: Unable to locate post ' + post_url +
                              ' nickname ' + nickname)
    return total_posts_count, posts_added_to_timeline


def _create_box_indexed(recent_posts_cache: {},
                        base_dir: str, boxname: str,
                        nickname: str, domain: str, port: int,
                        http_prefix: str,
                        items_per_page: int, header_only: bool,
                        authorized: bool,
                        newswire_votes_threshold: int, positive_voting: bool,
                        voting_time_mins: int, page_number: int,
                        first_post_id: str) -> {}:
    """Constructs the box feed for a person with the given nickname
    """
    if not authorized or not page_number:
        page_number = 1

    if boxname not in ('inbox', 'dm', 'tlreplies', 'tlmedia',
                       'tlblogs', 'tlnews', 'tlfeatures', 'outbox',
                       'tlbookmarks', 'bookmarks'):
        print('ERROR: invalid boxname ' + boxname)
        return None

    unauthorized_premium = False
    if not authorized and boxname == 'outbox':
        unauthorized_premium = is_premium_account(base_dir, nickname, domain)

    # bookmarks and events timelines are like the inbox
    # but have their own separate index
    index_box_name = boxname
    timeline_nickname = nickname
    if boxname == "tlbookmarks":
        boxname = "bookmarks"
        index_box_name = boxname
    elif boxname == "tlfeatures":
        boxname = "tlblogs"
        index_box_name = boxname
        timeline_nickname = 'news'

    original_domain = domain
    domain = get_full_domain(domain, port)

    box_actor = local_actor_url(http_prefix, nickname, domain)

    page_str = '?page=true'
    if page_number:
        page_number = max(page_number, 1)
        try:
            page_str = '?page=' + str(page_number)
        except BaseException:
            print('EX: _create_box_indexed ' +
                  'unable to convert page number to string')
    box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
    box_header = {
        '@context': 'https://www.w3.org/ns/activitystreams',
        'first': box_url + '?page=true',
        'id': box_url,
        'last': box_url + '?page=true',
        'totalItems': 0,
        'type': 'OrderedCollection'
    }
    box_items = {
        '@context': 'https://www.w3.org/ns/activitystreams',
        'id': box_url + page_str,
        'orderedItems': [
        ],
        'partOf': box_url,
        'type': 'OrderedCollectionPage'
    }

    posts_in_box = []
    post_urls_in_box = []

    if not unauthorized_premium:
        total_posts_count, posts_added_to_timeline = \
            _create_box_items(base_dir,
                              timeline_nickname,
                              original_domain,
                              nickname, domain,
                              index_box_name,
                              first_post_id,
                              page_number,
                              items_per_page,
                              newswire_votes_threshold,
                              positive_voting,
                              voting_time_mins,
                              post_urls_in_box,
                              recent_posts_cache,
                              boxname,
                              posts_in_box,
                              box_actor)
    else:
        # unauthorized requests for premium posts should return an empty set
        total_posts_count = posts_added_to_timeline = 0

    if first_post_id and \
       posts_added_to_timeline == 0 and \
       not unauthorized_premium:
        # no first post was found within the index, so just use the page number
        first_post_id = ''
        total_posts_count, posts_added_to_timeline = \
            _create_box_items(base_dir,
                              timeline_nickname,
                              original_domain,
                              nickname, domain,
                              index_box_name,
                              None,
                              page_number,
                              items_per_page,
                              newswire_votes_threshold,
                              positive_voting,
                              voting_time_mins,
                              post_urls_in_box,
                              recent_posts_cache,
                              boxname,
                              posts_in_box,
                              box_actor)

    if total_posts_count < 3:
        print('Posts added to json timeline ' + boxname + ': ' +
              str(posts_added_to_timeline))

    # Generate first and last entries within header
    if total_posts_count > 0:
        last_page = int(total_posts_count / items_per_page)
        last_page = max(last_page, 1)
        box_header['last'] = \
            local_actor_url(http_prefix, nickname, domain) + \
            '/' + boxname + '?page=' + str(last_page)

    if header_only:
        box_header['totalItems'] = len(posts_in_box)
        prev_page_str = 'true'
        if page_number > 1:
            prev_page_str = str(page_number - 1)
        box_header['prev'] = \
            local_actor_url(http_prefix, nickname, domain) + \
            '/' + boxname + '?page=' + prev_page_str

        next_page_str = str(page_number + 1)
        box_header['next'] = \
            local_actor_url(http_prefix, nickname, domain) + \
            '/' + boxname + '?page=' + next_page_str
        return box_header

    for post_str in posts_in_box:
        # Check if the post has replies
        has_replies = False
        if post_str.endswith('<hasReplies>'):
            has_replies = True
            # remove the replies identifier
            post_str = post_str.replace('<hasReplies>', '')

        # Check if the post was delivered via a third party
        mitm = False
        if post_str.endswith('<postmitm>'):
            mitm = True
            # remove the mitm identifier
            post_str = post_str.replace('<postmitm>', '')

        pst = None
        try:
            pst = json.loads(post_str)
        except BaseException:
            print('EX: _create_box_indexed unable to load json ' + post_str)
            continue

        # Does this post have replies?
        # This will be used to indicate that replies exist within the html
        # created by individual_post_as_html
        pst['hasReplies'] = has_replies

        # was the post delivered via a third party?
        pst['mitm'] = mitm

        if not authorized:
            if not remove_post_interactions(pst, False):
                continue

        box_items['orderedItems'].append(pst)

    return box_items


def expire_cache(base_dir: str, person_cache: {},
                 http_prefix: str, archive_dir: str,
                 recent_posts_cache: {},
                 max_posts_in_box: int,
                 max_cache_age_days: int):
    """Thread used to expire actors from the cache and archive old posts
    """
    while True:
        # once per day
        time.sleep(60 * 60 * 24)
        expire_person_cache(person_cache)
        archive_posts(base_dir, http_prefix, archive_dir, recent_posts_cache,
                      max_posts_in_box, max_cache_age_days)


def _expire_announce_cache_for_person(base_dir: str,
                                      nickname: str, domain: str,
                                      max_age_days: int) -> int:
    """Expires entries within the announces cache
    """
    cache_dir = base_dir + '/cache/announce/' + nickname
    if not os.path.isdir(cache_dir):
        print('No cached announces for ' + nickname + '@' + domain)
        return 0
    expired_post_count = 0
    posts_in_cache = os.scandir(cache_dir)
    for cache_filename in posts_in_cache:
        cache_filename = cache_filename.name
        # Time of file creation
        full_filename = os.path.join(cache_dir, cache_filename)
        if not os.path.isfile(full_filename):
            continue
        last_modified = file_last_modified(full_filename)
        # get time difference
        if not valid_post_date(last_modified, max_age_days, False):
            try:
                os.remove(full_filename)
            except OSError:
                print('EX: unable to delete from announce cache ' +
                      full_filename)
            expired_post_count += 1
    return expired_post_count


def _expire_posts_cache_for_person(base_dir: str,
                                   nickname: str, domain: str,
                                   max_age_days: int) -> int:
    """Expires entries within the posts cache
    """
    cache_dir = acct_dir(base_dir, nickname, domain) + '/postcache'
    if not os.path.isdir(cache_dir):
        print('No cached posts for ' + nickname + '@' + domain)
        return 0
    expired_post_count = 0
    posts_in_cache = os.scandir(cache_dir)
    for cache_filename in posts_in_cache:
        cache_filename = cache_filename.name
        # Time of file creation
        full_filename = os.path.join(cache_dir, cache_filename)
        if not os.path.isfile(full_filename):
            continue
        last_modified = file_last_modified(full_filename)
        # get time difference
        if not valid_post_date(last_modified, max_age_days, False):
            try:
                os.remove(full_filename)
            except OSError:
                print('EX: unable to delete from post cache ' +
                      full_filename)
            expired_post_count += 1
    return expired_post_count


def archive_posts(base_dir: str, http_prefix: str, archive_dir: str,
                  recent_posts_cache: {},
                  max_posts_in_box: int = 32000,
                  max_cache_age_days: int = 30) -> None:
    """Archives posts for all accounts
    """
    if max_posts_in_box == 0:
        return

    if archive_dir:
        if not os.path.isdir(archive_dir):
            os.mkdir(archive_dir)

    if archive_dir:
        if not os.path.isdir(archive_dir + '/accounts'):
            os.mkdir(archive_dir + '/accounts')

    for _, dirs, _ in os.walk(base_dir + '/accounts'):
        for handle in dirs:
            if '@' in handle:
                nickname = handle.split('@')[0]
                domain = handle.split('@')[1]
                archive_subdir = None
                if archive_dir:
                    archive_handle_dir = acct_handle_dir(archive_dir, handle)
                    if not os.path.isdir(archive_handle_dir):
                        os.mkdir(archive_handle_dir)
                    if not os.path.isdir(archive_handle_dir + '/inbox'):
                        os.mkdir(archive_handle_dir + '/inbox')
                    if not os.path.isdir(archive_handle_dir + '/outbox'):
                        os.mkdir(archive_handle_dir + '/outbox')
                    archive_subdir = archive_handle_dir + '/inbox'
                archive_posts_for_person(http_prefix,
                                         nickname, domain, base_dir,
                                         'inbox', archive_subdir,
                                         recent_posts_cache, max_posts_in_box)
                expired_announces = \
                    _expire_announce_cache_for_person(base_dir,
                                                      nickname, domain,
                                                      max_cache_age_days)
                print('Expired ' + str(expired_announces) +
                      ' cached announces for ' + nickname + '@' + domain)
                expired_posts = \
                    _expire_posts_cache_for_person(base_dir,
                                                   nickname, domain,
                                                   max_cache_age_days)
                print('Expired ' + str(expired_posts) +
                      ' cached posts for ' + nickname + '@' + domain)
                if archive_dir:
                    archive_subdir = archive_dir + '/accounts/' + \
                        handle + '/outbox'
                archive_posts_for_person(http_prefix,
                                         nickname, domain, base_dir,
                                         'outbox', archive_subdir,
                                         recent_posts_cache, max_posts_in_box)
        break


def _expire_posts_for_person(http_prefix: str, nickname: str, domain: str,
                             base_dir: str, recent_posts_cache: {},
                             max_age_days: int, debug: bool,
                             keep_dms: bool) -> int:
    """Removes posts older than some number of days
    """
    expired_post_count = 0
    if max_age_days <= 0:
        return expired_post_count

    boxname = 'outbox'
    box_dir = create_person_dir(nickname, domain, base_dir, boxname)

    posts_in_box = os.scandir(box_dir)
    for post_filename in posts_in_box:
        post_filename = post_filename.name
        if not post_filename.endswith('.json'):
            continue
        # Time of file creation
        full_filename = os.path.join(box_dir, post_filename)
        if not os.path.isfile(full_filename):
            continue
        content = ''
        try:
            with open(full_filename, 'r', encoding='utf-8') as fp_content:
                content = fp_content.read()
        except OSError:
            print('EX: expire_posts_for_person unable to open content ' +
                  full_filename)
        if '"published":' not in content:
            continue
        published_str = content.split('"published":')[1]
        if '"' not in published_str:
            continue
        published_str = published_str.split('"')[1]
        if not published_str.endswith('Z'):
            continue
        # get time difference
        if not valid_post_date(published_str, max_age_days, debug):
            if keep_dms:
                post_json_object = load_json(full_filename)
                if not post_json_object:
                    continue
                if is_dm(post_json_object):
                    continue
            delete_post(base_dir, http_prefix, nickname, domain,
                        full_filename, debug, recent_posts_cache, True)
            expired_post_count += 1

    return expired_post_count


def get_post_expiry_keep_dms(base_dir: str, nickname: str, domain: str) -> int:
    """Returns true if dms should expire
    """
    keep_dms = True
    handle = nickname + '@' + domain
    expire_dms_filename = \
        acct_handle_dir(base_dir, handle) + '/.expire_posts_dms'
    if os.path.isfile(expire_dms_filename):
        keep_dms = False
    return keep_dms


def set_post_expiry_keep_dms(base_dir: str, nickname: str, domain: str,
                             keep_dms: bool) -> None:
    """Sets whether to keep DMs during post expiry for an account
    """
    handle = nickname + '@' + domain
    expire_dms_filename = \
        acct_handle_dir(base_dir, handle) + '/.expire_posts_dms'
    if keep_dms:
        if os.path.isfile(expire_dms_filename):
            try:
                os.remove(expire_dms_filename)
            except OSError:
                print('EX: unable to write set_post_expiry_keep_dms False ' +
                      expire_dms_filename)
        return
    try:
        with open(expire_dms_filename, 'w+', encoding='utf-8') as fp_expire:
            fp_expire.write('\n')
    except OSError:
        print('EX: unable to write set_post_expiry_keep_dms True ' +
              expire_dms_filename)


def expire_posts(base_dir: str, http_prefix: str,
                 recent_posts_cache: {}, debug: bool) -> int:
    """Expires posts for instance accounts
    """
    expired_post_count = 0
    for _, dirs, _ in os.walk(base_dir + '/accounts'):
        for handle in dirs:
            if '@' not in handle:
                continue
            nickname = handle.split('@')[0]
            domain = handle.split('@')[1]
            expire_posts_filename = \
                acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
            if not os.path.isfile(expire_posts_filename):
                continue
            keep_dms = get_post_expiry_keep_dms(base_dir, nickname, domain)
            expire_days_str = None
            try:
                with open(expire_posts_filename, 'r',
                          encoding='utf-8') as fp_expire:
                    expire_days_str = fp_expire.read()
            except OSError:
                print('EX: expire_posts failed to read days file ' +
                      expire_posts_filename)
                continue
            if not expire_days_str:
                continue
            if not expire_days_str.isdigit():
                continue
            max_age_days = int(expire_days_str)
            if max_age_days <= 0:
                continue
            expired_post_count += \
                _expire_posts_for_person(http_prefix,
                                         nickname, domain, base_dir,
                                         recent_posts_cache,
                                         max_age_days, debug,
                                         keep_dms)
        break
    return expired_post_count


def get_post_expiry_days(base_dir: str, nickname: str, domain: str) -> int:
    """Returns the post expiry period for the given account
    """
    handle = nickname + '@' + domain
    expire_posts_filename = \
        acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
    if not os.path.isfile(expire_posts_filename):
        return 0
    days_str = None
    try:
        with open(expire_posts_filename, 'r', encoding='utf-8') as fp_expire:
            days_str = fp_expire.read()
    except OSError:
        print('EX: unable to write post expire days ' +
              expire_posts_filename)
    if not days_str:
        return 0
    if not days_str.isdigit():
        return 0
    return int(days_str)


def set_post_expiry_days(base_dir: str, nickname: str, domain: str,
                         max_age_days: int) -> None:
    """Sets the number of days after which posts from an account will expire
    """
    handle = nickname + '@' + domain
    expire_posts_filename = \
        acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
    try:
        with open(expire_posts_filename, 'w+', encoding='utf-8') as fp_expire:
            fp_expire.write(str(max_age_days))
    except OSError:
        print('EX: unable to write post expire days ' +
              expire_posts_filename)


def archive_posts_for_person(http_prefix: str, nickname: str, domain: str,
                             base_dir: str,
                             boxname: str, archive_dir: str,
                             recent_posts_cache: {},
                             max_posts_in_box=32000) -> None:
    """Retain a maximum number of posts within the given box
    Move any others to an archive directory
    """
    if boxname not in ('inbox', 'outbox'):
        return
    if archive_dir:
        if not os.path.isdir(archive_dir):
            os.mkdir(archive_dir)
    box_dir = create_person_dir(nickname, domain, base_dir, boxname)
    posts_in_box = os.scandir(box_dir)
    no_of_posts = 0
    for _ in posts_in_box:
        no_of_posts += 1
    if no_of_posts <= max_posts_in_box:
        print('Checked ' + str(no_of_posts) + ' ' + boxname +
              ' posts for ' + nickname + '@' + domain)
        return

    # remove entries from the index
    handle = nickname + '@' + domain
    index_filename = \
        acct_handle_dir(base_dir, handle) + '/' + boxname + '.index'
    if os.path.isfile(index_filename):
        index_ctr = 0
        # get the existing index entries as a string
        new_index = ''
        try:
            with open(index_filename, 'r', encoding='utf-8') as index_file:
                for post_id in index_file:
                    new_index += post_id
                    index_ctr += 1
                    if index_ctr >= max_posts_in_box:
                        break
        except OSError as ex:
            print('EX: archive_posts_for_person unable to read ' +
                  index_filename + ' ' + str(ex))
        # save the new index file
        if len(new_index) > 0:
            try:
                with open(index_filename, 'w+',
                          encoding='utf-8') as index_file:
                    index_file.write(new_index)
            except OSError:
                print('EX: archive_posts_for_person unable to write ' +
                      index_filename)

    posts_in_box_dict = {}
    posts_ctr = 0
    posts_in_box = os.scandir(box_dir)
    for post_filename in posts_in_box:
        post_filename = post_filename.name
        if not post_filename.endswith('.json'):
            continue
        # Time of file creation
        full_filename = os.path.join(box_dir, post_filename)
        if os.path.isfile(full_filename):
            content = ''
            try:
                with open(full_filename, 'r', encoding='utf-8') as fp_content:
                    content = fp_content.read()
            except OSError:
                print('EX: unable to open content ' + full_filename)
            if '"published":' in content:
                published_str = content.split('"published":')[1]
                if '"' in published_str:
                    published_str = published_str.split('"')[1]
                    if published_str.endswith('Z'):
                        posts_in_box_dict[published_str] = post_filename
                        posts_ctr += 1

    no_of_posts = posts_ctr
    if no_of_posts <= max_posts_in_box:
        print('Checked ' + str(no_of_posts) + ' ' + boxname +
              ' posts for ' + nickname + '@' + domain)
        return

    # sort the list in ascending order of date
    posts_in_box_sorted = \
        OrderedDict(sorted(posts_in_box_dict.items(), reverse=False))

    # directory containing cached html posts
    post_cache_dir = box_dir.replace('/' + boxname, '/postcache')

    remove_ctr = 0
    for published_str, post_filename in posts_in_box_sorted.items():
        file_path = os.path.join(box_dir, post_filename)
        if not os.path.isfile(file_path):
            continue
        if archive_dir:
            archive_path = os.path.join(archive_dir, post_filename)
            os.rename(file_path, archive_path)

            extensions = ('replies', 'votes', 'arrived', 'muted')
            for ext in extensions:
                ext_path = file_path.replace('.json', '.' + ext)
                if os.path.isfile(ext_path):
                    os.rename(ext_path,
                              archive_path.replace('.json', '.' + ext))
                else:
                    ext_path = file_path.replace('.json',
                                                 '.json.' + ext)
                    if os.path.isfile(ext_path):
                        os.rename(ext_path,
                                  archive_path.replace('.json',
                                                       '.json.' + ext))
        else:
            delete_post(base_dir, http_prefix, nickname, domain,
                        file_path, False, recent_posts_cache, False)

        # remove cached html posts
        post_cache_filename = \
            os.path.join(post_cache_dir, post_filename)
        post_cache_filename = post_cache_filename.replace('.json', '.html')
        if os.path.isfile(post_cache_filename):
            try:
                os.remove(post_cache_filename)
            except OSError:
                print('EX: archive_posts_for_person unable to delete ' +
                      post_cache_filename)

        no_of_posts -= 1
        remove_ctr += 1
        if no_of_posts <= max_posts_in_box:
            break
    if archive_dir:
        print('Archived ' + str(remove_ctr) + ' ' + boxname +
              ' posts for ' + nickname + '@' + domain)
    else:
        print('Removed ' + str(remove_ctr) + ' ' + boxname +
              ' posts for ' + nickname + '@' + domain)
    print(nickname + '@' + domain + ' has ' + str(no_of_posts) +
          ' in ' + boxname)


def get_public_posts_of_person(base_dir: str, nickname: str, domain: str,
                               raw: bool, simple: bool, proxy_type: str,
                               port: int, http_prefix: str,
                               debug: bool, project_version: str,
                               system_language: str,
                               signing_priv_key_pem: str,
                               origin_domain: str) -> None:
    """ This is really just for test purposes
    """
    if debug:
        if signing_priv_key_pem:
            print('Signing key available')
        else:
            print('Signing key missing')

    if debug:
        print('Starting new session for getting public posts')
    session = create_session(proxy_type)
    if not session:
        if debug:
            print('Session was not created')
        return
    person_cache = {}
    cached_webfingers = {}
    federation_list = []
    group_account = False
    if nickname.startswith('!'):
        nickname = nickname[1:]
        group_account = True
    domain_full = get_full_domain(domain, port)
    handle = http_prefix + "://" + domain_full + "/@" + nickname

    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         origin_domain, project_version, debug, group_account,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('No webfinger result was returned for ' + handle)
        sys.exit()
    if not isinstance(wf_request, dict):
        print('Webfinger for ' + handle + ' did not return a dict. ' +
              str(wf_request))
        sys.exit()

    if debug:
        print('\nGetting the outbox for ' + handle)
    (person_url, _, _, person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            nickname, domain, 'outbox',
                            62524, system_language)
    if debug:
        print('Actor url: ' + str(person_id))
    if not person_id:
        return

    max_mentions = 10
    max_emoji = 10
    max_attachments = 5
    _get_posts(session, person_url, 30, max_mentions, max_emoji,
               max_attachments, federation_list, raw, simple, debug,
               project_version, http_prefix, origin_domain, system_language,
               signing_priv_key_pem)


def get_public_post_domains(session, base_dir: str, nickname: str, domain: str,
                            origin_domain: str,
                            proxy_type: str, port: int, http_prefix: str,
                            debug: bool, project_version: str,
                            word_frequency: {}, domain_list: [],
                            system_language: str,
                            signing_priv_key_pem: str) -> []:
    """ Returns a list of domains referenced within public posts
    """
    if not session:
        session = create_session(proxy_type)
    if not session:
        return domain_list
    person_cache = {}
    cached_webfingers = {}

    domain_full = get_full_domain(domain, port)
    handle = http_prefix + "://" + domain_full + "/@" + nickname
    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        return domain_list
    if not isinstance(wf_request, dict):
        print('Webfinger for ' + handle + ' did not return a dict. ' +
              str(wf_request))
        return domain_list

    (person_url, _, _, _, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            nickname, domain, 'outbox',
                            92522, system_language)
    post_domains = \
        get_post_domains(session, person_url, 64, debug,
                         project_version, http_prefix, domain,
                         word_frequency, domain_list, system_language,
                         signing_priv_key_pem)
    post_domains.sort()
    return post_domains


def download_follow_collection(signing_priv_key_pem: str,
                               follow_type: str,
                               session, http_prefix: str,
                               actor: str, page_number: int,
                               no_of_pages: int, debug: bool) -> []:
    """Returns a list of following/followers for the given actor
    by downloading the json for their following/followers collection
    """
    prof = 'https://www.w3.org/ns/activitystreams'
    if '/channel/' not in actor or '/accounts/' not in actor:
        accept_str = \
            'application/activity+json; ' + \
            'profile="' + prof + '"'
        session_headers = {
            'Accept': accept_str
        }
    else:
        accept_str = \
            'application/ld+json; ' + \
            'profile="' + prof + '"'
        session_headers = {
            'Accept': accept_str
        }
    result = []
    for page_ctr in range(no_of_pages):
        url = \
            actor + '/' + follow_type + '?page=' + str(page_number + page_ctr)
        followers_json = \
            get_json(signing_priv_key_pem, session, url, session_headers, None,
                     debug, __version__, http_prefix, None)
        if get_json_valid(followers_json):
            if followers_json.get('orderedItems'):
                for follower_actor in followers_json['orderedItems']:
                    if follower_actor not in result:
                        result.append(follower_actor)
            elif followers_json.get('items'):
                for follower_actor in followers_json['items']:
                    if follower_actor not in result:
                        result.append(follower_actor)
            else:
                break
        else:
            break
    return result


def get_public_post_info(session, base_dir: str, nickname: str, domain: str,
                         origin_domain: str,
                         proxy_type: str, port: int, http_prefix: str,
                         debug: bool, project_version: str,
                         word_frequency: {}, system_language: str,
                         signing_priv_key_pem: str) -> []:
    """ Returns a dict of domains referenced within public posts
    """
    if not session:
        session = create_session(proxy_type)
    if not session:
        return {}
    person_cache = {}
    cached_webfingers = {}

    domain_full = get_full_domain(domain, port)
    handle = http_prefix + "://" + domain_full + "/@" + nickname
    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        return {}
    if not isinstance(wf_request, dict):
        print('Webfinger for ' + handle + ' did not return a dict. ' +
              str(wf_request))
        return {}

    (person_url, _, _, _, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            nickname, domain, 'outbox',
                            13863, system_language)
    max_posts = 64
    post_domains = \
        get_post_domains(session, person_url, max_posts, debug,
                         project_version, http_prefix, domain,
                         word_frequency, [], system_language,
                         signing_priv_key_pem)
    post_domains.sort()
    domains_info = {}
    for pdomain in post_domains:
        if not domains_info.get(pdomain):
            domains_info[pdomain] = []

    blocked_posts = \
        _get_posts_for_blocked_domains(base_dir, session,
                                       person_url, max_posts,
                                       debug,
                                       project_version, http_prefix,
                                       domain, signing_priv_key_pem)
    for blocked_domain, post_url_list in blocked_posts.items():
        domains_info[blocked_domain] += post_url_list

    return domains_info


def get_public_post_domains_blocked(session, base_dir: str,
                                    nickname: str, domain: str,
                                    proxy_type: str, port: int,
                                    http_prefix: str,
                                    debug: bool, project_version: str,
                                    word_frequency: {}, domain_list: [],
                                    system_language: str,
                                    signing_priv_key_pem: str) -> []:
    """ Returns a list of domains referenced within public posts which
    are globally blocked on this instance
    """
    origin_domain = domain
    post_domains = \
        get_public_post_domains(session, base_dir, nickname, domain,
                                origin_domain,
                                proxy_type, port, http_prefix,
                                debug, project_version,
                                word_frequency, domain_list, system_language,
                                signing_priv_key_pem)
    if not post_domains:
        return []

    blocking_filename = base_dir + '/accounts/blocking.txt'
    if not os.path.isfile(blocking_filename):
        return []

    # read the blocked domains as a single string
    blocked_str = ''
    with open(blocking_filename, 'r', encoding='utf-8') as fp_block:
        blocked_str = fp_block.read()

    blocked_domains = []
    for domain_name in post_domains:
        if '@' not in domain_name:
            continue
        # get the domain after the @
        domain_name = domain_name.split('@')[1].strip()
        if is_evil(domain_name):
            blocked_domains.append(domain_name)
            continue
        if domain_name in blocked_str:
            blocked_domains.append(domain_name)

    return blocked_domains


def _get_non_mutuals_of_person(base_dir: str,
                               nickname: str, domain: str) -> []:
    """Returns the followers who are not mutuals of a person
    i.e. accounts which follow you but you don't follow them
    """
    followers = \
        get_followers_list(base_dir, nickname, domain, 'followers.txt')
    following = \
        get_followers_list(base_dir, nickname, domain, 'following.txt')
    non_mutuals = []
    for handle in followers:
        if handle not in following:
            non_mutuals.append(handle)
    return non_mutuals


def check_domains(session, base_dir: str,
                  nickname: str, domain: str,
                  proxy_type: str, port: int, http_prefix: str,
                  debug: bool, project_version: str,
                  max_blocked_domains: int, single_check: bool,
                  system_language: str,
                  signing_priv_key_pem: str) -> None:
    """Checks follower accounts for references to globally blocked domains
    """
    word_frequency = {}
    non_mutuals = _get_non_mutuals_of_person(base_dir, nickname, domain)
    if not non_mutuals:
        print('No non-mutual followers were found')
        return
    follower_warning_filename = base_dir + '/accounts/followerWarnings.txt'
    update_follower_warnings = False
    follower_warning_str = ''
    if os.path.isfile(follower_warning_filename):
        with open(follower_warning_filename, 'r',
                  encoding='utf-8') as fp_warn:
            follower_warning_str = fp_warn.read()

    if single_check:
        # checks a single random non-mutual
        index = random.randrange(0, len(non_mutuals))
        handle = non_mutuals[index]
        if '@' in handle:
            non_mutual_nickname = handle.split('@')[0]
            non_mutual_domain = handle.split('@')[1].strip()
            blocked_domains = \
                get_public_post_domains_blocked(session, base_dir,
                                                non_mutual_nickname,
                                                non_mutual_domain,
                                                proxy_type, port, http_prefix,
                                                debug, project_version,
                                                word_frequency, [],
                                                system_language,
                                                signing_priv_key_pem)
            if blocked_domains:
                if len(blocked_domains) > max_blocked_domains:
                    follower_warning_str += handle + '\n'
                    update_follower_warnings = True
    else:
        # checks all non-mutuals
        for handle in non_mutuals:
            if '@' not in handle:
                continue
            if handle in follower_warning_str:
                continue
            non_mutual_nickname = handle.split('@')[0]
            non_mutual_domain = handle.split('@')[1].strip()
            blocked_domains = \
                get_public_post_domains_blocked(session, base_dir,
                                                non_mutual_nickname,
                                                non_mutual_domain,
                                                proxy_type, port, http_prefix,
                                                debug, project_version,
                                                word_frequency, [],
                                                system_language,
                                                signing_priv_key_pem)
            if blocked_domains:
                print(handle)
                for bdomain in blocked_domains:
                    print('  ' + bdomain)
                if len(blocked_domains) > max_blocked_domains:
                    follower_warning_str += handle + '\n'
                    update_follower_warnings = True

    if update_follower_warnings and follower_warning_str:
        try:
            with open(follower_warning_filename, 'w+',
                      encoding='utf-8') as fp_warn:
                fp_warn.write(follower_warning_str)
        except OSError:
            print('EX: check_domains unable to write ' +
                  follower_warning_filename)
        if not single_check:
            print(follower_warning_str)


def populate_replies_json(base_dir: str, nickname: str, domain: str,
                          post_replies_filename: str, authorized: bool,
                          replies_json: {}) -> None:
    pub_str = 'https://www.w3.org/ns/activitystreams#Public'
    # populate the items list with replies
    replies_boxes = ('outbox', 'inbox')
    with open(post_replies_filename, 'r', encoding='utf-8') as replies_file:
        for message_id in replies_file:
            reply_found = False
            # examine inbox and outbox
            for boxname in replies_boxes:
                message_id2 = remove_eol(message_id)
                search_filename = \
                    acct_dir(base_dir, nickname, domain) + '/' + \
                    boxname + '/' + \
                    message_id2.replace('/', '#') + '.json'
                if os.path.isfile(search_filename):
                    if authorized or \
                       text_in_file(pub_str, search_filename):
                        post_json_object = load_json(search_filename)
                        if post_json_object:
                            if post_json_object['object'].get('cc'):
                                pjo = post_json_object
                                if (authorized or
                                    (pub_str in pjo['object']['to'] or
                                     pub_str in pjo['object']['cc'])):
                                    replies_json['orderedItems'].append(pjo)
                                    reply_found = True
                            else:
                                if authorized or \
                                   pub_str in post_json_object['object']['to']:
                                    pjo = post_json_object
                                    replies_json['orderedItems'].append(pjo)
                                    reply_found = True
                    break
            # if not in either inbox or outbox then examine the shared inbox
            if not reply_found:
                message_id2 = remove_eol(message_id)
                search_filename = \
                    base_dir + \
                    '/accounts/inbox@' + \
                    domain + '/inbox/' + \
                    message_id2.replace('/', '#') + '.json'
                if os.path.isfile(search_filename):
                    if authorized or \
                       text_in_file(pub_str, search_filename):
                        # get the json of the reply and append it to
                        # the collection
                        post_json_object = load_json(search_filename)
                        if post_json_object:
                            if post_json_object['object'].get('cc'):
                                pjo = post_json_object
                                if (authorized or
                                    (pub_str in pjo['object']['to'] or
                                     pub_str in pjo['object']['cc'])):
                                    pjo = post_json_object
                                    replies_json['orderedItems'].append(pjo)
                            else:
                                if authorized or \
                                   pub_str in post_json_object['object']['to']:
                                    pjo = post_json_object
                                    replies_json['orderedItems'].append(pjo)


def _reject_announce(announce_filename: str,
                     base_dir: str, nickname: str, domain: str,
                     announce_post_id: str, recent_posts_cache: {}):
    """Marks an announce as rejected
    """
    reject_post_id(base_dir, nickname, domain, announce_post_id,
                   recent_posts_cache)

    # reject the post referenced by the announce activity object
    if not os.path.isfile(announce_filename + '.reject'):
        try:
            with open(announce_filename + '.reject', 'w+',
                      encoding='utf-8') as reject_announce_file:
                reject_announce_file.write('\n')
        except OSError:
            print('EX: _reject_announce unable to write ' +
                  announce_filename + '.reject')


def download_announce(session, base_dir: str, http_prefix: str,
                      nickname: str, domain: str,
                      post_json_object: {}, project_version: str,
                      yt_replace_domain: str,
                      twitter_replacement_domain: str,
                      allow_local_network_access: bool,
                      recent_posts_cache: {}, debug: bool,
                      system_language: str,
                      domain_full: str, person_cache: {},
                      signing_priv_key_pem: str,
                      blocked_cache: {}, block_federated: [],
                      bold_reading: bool,
                      show_vote_posts: bool,
                      languages_understood: []) -> {}:
    """Download the post referenced by an announce
    """
    if not post_json_object.get('object'):
        return None
    if not isinstance(post_json_object['object'], str):
        return None
    # ignore self-boosts
    actor_url = get_actor_from_post(post_json_object)
    if actor_url in post_json_object['object']:
        return None

    # get the announced post
    announce_cache_dir = base_dir + '/cache/announce/' + nickname
    if not os.path.isdir(announce_cache_dir):
        os.mkdir(announce_cache_dir)

    post_id = None
    if post_json_object.get('id'):
        post_id = remove_id_ending(post_json_object['id'])
    announce_filename = \
        announce_cache_dir + '/' + \
        post_json_object['object'].replace('/', '#') + '.json'

    if os.path.isfile(announce_filename + '.reject'):
        return None

    if os.path.isfile(announce_filename):
        if debug:
            print('Reading cached Announce content for ' +
                  post_json_object['object'])
        post_json_object = load_json(announce_filename)
        if post_json_object:
            return post_json_object
    else:
        profile_str = 'https://www.w3.org/ns/activitystreams'
        accept_str = \
            'application/activity+json; ' + \
            'profile="' + profile_str + '"'
        as_header = {
            'Accept': accept_str
        }
        if '/channel/' in actor_url or \
           '/accounts/' in actor_url:
            accept_str = \
                'application/ld+json; ' + \
                'profile="' + profile_str + '"'
            as_header = {
                'Accept': accept_str
            }
        actor_nickname = get_nickname_from_actor(actor_url)
        if not actor_nickname:
            print('WARN: download_announce no actor_nickname')
            return None
        actor_domain, actor_port = \
            get_domain_from_actor(actor_url)
        if not actor_domain:
            print('Announce actor does not contain a ' +
                  'valid domain or port number: ' +
                  actor_url)
            return None
        if is_blocked(base_dir, nickname, domain,
                      actor_nickname, actor_domain,
                      None, block_federated):
            print('Announce download blocked actor: ' +
                  actor_nickname + '@' + actor_domain)
            return None
        object_nickname = get_nickname_from_actor(post_json_object['object'])
        object_domain, _ = \
            get_domain_from_actor(post_json_object['object'])
        if not object_domain:
            print('Announce object does not contain a ' +
                  'valid domain or port number: ' +
                  str(post_json_object['object']))
            return None
        if is_blocked(base_dir, nickname, domain, object_nickname,
                      object_domain, None, block_federated):
            if object_nickname and object_domain:
                print('Announce download blocked handle: ' +
                      object_nickname + '@' + object_domain)
            else:
                print('Announce download blocked handle 2: ' +
                      str(post_json_object['object']))
            return None
        if debug:
            print('Downloading Announce content for ' +
                  post_json_object['object'])
        announced_json = \
            get_json(signing_priv_key_pem, session,
                     post_json_object['object'],
                     as_header, None, debug, project_version,
                     http_prefix, domain)

        if not get_json_valid(announced_json):
            return None

        if not isinstance(announced_json, dict):
            print('WARN: announced post json is not a dict - ' +
                  post_json_object['object'] + ' ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if announced_json.get('error'):
            print('WARN: ' +
                  'Attempt to download announce returned an error ' +
                  post_json_object['object'] + ' ' +
                  str(announced_json))
            return None
        if not announced_json.get('id'):
            print('WARN: announced post does not have an id ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        announced_actor = announced_json['id']
        if announced_json.get('attributedTo'):
            announced_actor = get_attributed_to(announced_json['attributedTo'])
        if not announced_json.get('type'):
            print('WARN: announced post does not have a type ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if announced_json['type'] == 'Video':
            converted_json = \
                convert_video_to_note(base_dir, nickname, domain,
                                      system_language,
                                      announced_json, blocked_cache,
                                      block_federated,
                                      languages_understood)
            if converted_json:
                announced_json = converted_json
        if not contains_statuses(announced_json['id']):
            print('WARN: announced post id does not contain /statuses/ ' +
                  'or /objects/ or /p/ ' + str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if not has_users_path(announced_actor):
            print('WARN: announced post id does not contain /users/ ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if announced_json['type'] not in ('Note', 'Event', 'Page',
                                          'Question', 'Article'):
            print('WARN: announced post is not Note/Page/Article/Question ' +
                  str(announced_json))
            # You can only announce Note, Page, Article or Question types
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if announced_json['type'] == 'Question':
            if not announced_json.get('endTime') or \
               not announced_json.get('oneOf'):
                # announced Question should have an end time
                # and options list
                print('WARN: announced Question should have endTime ' +
                      str(announced_json))
                _reject_announce(announce_filename,
                                 base_dir, nickname, domain, post_id,
                                 recent_posts_cache)
                return None
            if not isinstance(announced_json['oneOf'], list):
                print('WARN: announced Question oneOf should be a list ' +
                      str(announced_json))
                _reject_announce(announce_filename,
                                 base_dir, nickname, domain, post_id,
                                 recent_posts_cache)
                return None
            if is_question_filtered(base_dir, nickname, domain,
                                    system_language, announced_json):
                if announced_json.get('id'):
                    print('REJECT: announced question was filtered ' +
                          str(announced_json['id']))
                _reject_announce(announce_filename,
                                 base_dir, nickname, domain, post_id,
                                 recent_posts_cache)
                return None
        if 'content' not in announced_json:
            print('WARN: announced post does not have content ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if not announced_json.get('published'):
            print('WARN: announced post does not have published ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if '.' in announced_json['published'] and \
           'Z' in announced_json['published']:
            announced_json['published'] = \
                announced_json['published'].split('.')[0] + 'Z'
        if not valid_post_date(announced_json['published'], 90, debug):
            print('WARN: announced post is not recently published ' +
                  str(announced_json['published']))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if not understood_post_language(base_dir, nickname,
                                        announced_json, system_language,
                                        http_prefix, domain_full,
                                        person_cache):
            return None
        # Check the content of the announce
        convert_post_content_to_html(announced_json)
        content_str = announced_json['content']
        using_content_map = False
        if 'contentMap' in announced_json:
            if announced_json['contentMap'].get(system_language):
                content_str = announced_json['contentMap'][system_language]
                using_content_map = True
        if dangerous_markup(content_str, allow_local_network_access, []):
            print('WARN: announced post contains dangerous markup ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        summary_str = \
            get_summary_from_post(announced_json, system_language, [])
        media_descriptions = \
            get_media_descriptions_from_post(announced_json)
        content_all = content_str
        if summary_str:
            content_all = \
                summary_str + ' ' + content_str + ' ' + media_descriptions
        if is_filtered(base_dir, nickname, domain, content_all,
                       system_language):
            if announced_json.get('id'):
                print('REJECT: announced post has been filtered ' +
                      str(announced_json['id']))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if reject_twitter_summary(base_dir, nickname, domain,
                                  summary_str):
            print('WARN: announced post has twitter summary ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if contains_private_key(content_str):
            print("WARN: announced post contains someone's private key " +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None
        if invalid_ciphertext(content_str):
            print('WARN: announced post contains invalid ciphertext ' +
                  str(announced_json))
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None

        # remove any long words
        content_str = remove_long_words(content_str, 40, [])

        # Prevent the same word from being repeated many times
        content_str = limit_repeated_words(content_str, 6)

        # remove text formatting, such as bold/italics
        content_str = remove_text_formatting(content_str, bold_reading)

        # set the content after santitization
        if using_content_map:
            announced_json['contentMap'][system_language] = content_str
        announced_json['content'] = content_str

        # wrap in create to be consistent with other posts
        announced_json = \
            outbox_message_create_wrap(http_prefix,
                                       actor_nickname, actor_domain,
                                       actor_port, announced_json)
        if announced_json['type'] != 'Create':
            print('WARN: announced post could not be wrapped in Create ' +
                  str(announced_json))
            # Create wrap failed
            _reject_announce(announce_filename,
                             base_dir, nickname, domain, post_id,
                             recent_posts_cache)
            return None

        # if poll/vote/question is not to be shown
        if not show_vote_posts:
            if is_question(announced_json):
                return None

        # set the id to the original status
        announced_json['id'] = post_json_object['object']
        announced_json['object']['id'] = post_json_object['object']
        # check that the repeat isn't for a blocked account
        attributed_nickname = \
            get_nickname_from_actor(announced_json['object']['id'])
        attributed_domain, attributed_port = \
            get_domain_from_actor(announced_json['object']['id'])
        if attributed_nickname and attributed_domain:
            attributed_domain = \
                get_full_domain(attributed_domain, attributed_port)
            if is_blocked(base_dir, nickname, domain,
                          attributed_nickname, attributed_domain,
                          None, block_federated):
                print('WARN: announced post handle is blocked ' +
                      str(attributed_nickname) + '@' + attributed_domain)
                _reject_announce(announce_filename,
                                 base_dir, nickname, domain, post_id,
                                 recent_posts_cache)
                return None
        post_json_object = announced_json
        replace_you_tube(post_json_object, yt_replace_domain, system_language)
        replace_twitter(post_json_object, twitter_replacement_domain,
                        system_language)
        if save_json(post_json_object, announce_filename):
            return post_json_object
    return None


def is_muted_conv(base_dir: str, nickname: str, domain: str, post_id: str,
                  conversation_id: str) -> bool:
    """Returns true if the given post is muted
    """
    if conversation_id:
        conv_muted_filename = \
            acct_dir(base_dir, nickname, domain) + '/conversation/' + \
            conversation_id.replace('/', '#') + '.muted'
        if os.path.isfile(conv_muted_filename):
            return True
    post_filename = locate_post(base_dir, nickname, domain, post_id)
    if not post_filename:
        return False
    if os.path.isfile(post_filename + '.muted'):
        return True
    return False


def send_block_via_server(base_dir: str, session,
                          from_nickname: str, password: str,
                          from_domain: str, from_port: int,
                          http_prefix: str, blocked_url: str,
                          cached_webfingers: {}, person_cache: {},
                          debug: bool, project_version: str,
                          signing_priv_key_pem: str,
                          system_language: str) -> {}:
    """Creates a block via c2s
    """
    if not session:
        print('WARN: No session for send_block_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, from_port)

    block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = block_actor + '/followers'

    new_block_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Block',
        'actor': block_actor,
        'object': blocked_url,
        'to': [to_url],
        'cc': [cc_url]
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  from_domain, project_version, debug, False,
                                  signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: block webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: block Webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            from_nickname,
                            from_domain, post_to_box, 72652,
                            system_language)

    if not inbox_url:
        if debug:
            print('DEBUG: block no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: block no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = post_json(http_prefix, from_domain_full,
                            session, new_block_json, [], inbox_url,
                            headers, 30, True)
    if not post_result:
        print('WARN: block unable to post')

    if debug:
        print('DEBUG: c2s POST block success')

    return new_block_json


def send_mute_via_server(base_dir: str, session,
                         from_nickname: str, password: str,
                         from_domain: str, from_port: int,
                         http_prefix: str, muted_url: str,
                         cached_webfingers: {}, person_cache: {},
                         debug: bool, project_version: str,
                         signing_priv_key_pem: str,
                         system_language: str) -> {}:
    """Creates a mute via c2s
    """
    if not session:
        print('WARN: No session for send_mute_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, from_port)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    handle = replace_users_with_at(actor)

    new_mute_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Ignore',
        'actor': actor,
        'to': [actor],
        'object': muted_url
    }

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  from_domain, project_version, debug, False,
                                  signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: mute webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: mute Webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            from_nickname,
                            from_domain, post_to_box, 72652,
                            system_language)

    if not inbox_url:
        if debug:
            print('DEBUG: mute no ' + post_to_box + ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: mute no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = post_json(http_prefix, from_domain_full,
                            session, new_mute_json, [], inbox_url,
                            headers, 3, True)
    if post_result is None:
        print('WARN: mute unable to post')

    if debug:
        print('DEBUG: c2s POST mute success')

    return new_mute_json


def send_undo_mute_via_server(base_dir: str, session,
                              from_nickname: str, password: str,
                              from_domain: str, from_port: int,
                              http_prefix: str, muted_url: str,
                              cached_webfingers: {}, person_cache: {},
                              debug: bool, project_version: str,
                              signing_priv_key_pem: str,
                              system_language: str) -> {}:
    """Undoes a mute via c2s
    """
    if not session:
        print('WARN: No session for send_undo_mute_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, from_port)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    handle = replace_users_with_at(actor)

    undo_mute_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Undo',
        'actor': actor,
        'to': [actor],
        'object': {
            'type': 'Ignore',
            'actor': actor,
            'to': [actor],
            'object': muted_url
        }
    }

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  from_domain, project_version, debug, False,
                                  signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: undo mute webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: undo mute Webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            from_nickname,
                            from_domain, post_to_box, 72652,
                            system_language)

    if not inbox_url:
        if debug:
            print('DEBUG: undo mute no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: undo mute no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = post_json(http_prefix, from_domain_full,
                            session, undo_mute_json, [], inbox_url,
                            headers, 3, True)
    if post_result is None:
        print('WARN: undo mute unable to post')

    if debug:
        print('DEBUG: c2s POST undo mute success')

    return undo_mute_json


def send_undo_block_via_server(base_dir: str, session,
                               from_nickname: str, password: str,
                               from_domain: str, from_port: int,
                               http_prefix: str, blocked_url: str,
                               cached_webfingers: {}, person_cache: {},
                               debug: bool, project_version: str,
                               signing_priv_key_pem: str,
                               system_language: str) -> {}:
    """Creates a block via c2s
    """
    if not session:
        print('WARN: No session for send_block_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, from_port)

    block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = block_actor + '/followers'

    new_block_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Undo',
        'actor': block_actor,
        'object': {
            'type': 'Block',
            'actor': block_actor,
            'object': blocked_url,
            'to': [to_url],
            'cc': [cc_url]
        }
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = webfinger_handle(session, handle, http_prefix,
                                  cached_webfingers,
                                  from_domain, project_version, debug, False,
                                  signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: unblock webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: unblock webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     _, _) = get_person_box(signing_priv_key_pem,
                            origin_domain,
                            base_dir, session, wf_request,
                            person_cache,
                            project_version, http_prefix,
                            from_nickname,
                            from_domain, post_to_box, 53892,
                            system_language)

    if not inbox_url:
        if debug:
            print('DEBUG: unblock no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: unblock no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = post_json(http_prefix, from_domain_full,
                            session, new_block_json, [], inbox_url,
                            headers, 30, True)
    if not post_result:
        print('WARN: unblock unable to post')

    if debug:
        print('DEBUG: c2s POST unblock success')

    return new_block_json


def post_is_muted(base_dir: str, nickname: str, domain: str,
                  post_json_object: {}, message_id: str) -> bool:
    """ Returns true if the given post is muted
    """
    is_muted = None
    if 'muted' in post_json_object:
        is_muted = post_json_object['muted']
    if is_muted is True or is_muted is False:
        return is_muted

    is_muted = False
    post_dir = acct_dir(base_dir, nickname, domain)
    mute_filename = \
        post_dir + '/inbox/' + message_id.replace('/', '#') + '.json.muted'
    if os.path.isfile(mute_filename):
        is_muted = True
    else:
        mute_filename = \
            post_dir + '/outbox/' + \
            message_id.replace('/', '#') + '.json.muted'
        if os.path.isfile(mute_filename):
            is_muted = True
        else:
            mute_filename = \
                base_dir + '/accounts/cache/announce/' + nickname + \
                '/' + message_id.replace('/', '#') + '.json.muted'
            if os.path.isfile(mute_filename):
                is_muted = True
    return is_muted


def c2s_box_json(session, nickname: str, password: str,
                 domain: str, port: int,
                 http_prefix: str,
                 box_name: str, page_number: int,
                 debug: bool, signing_priv_key_pem: str) -> {}:
    """C2S Authenticated GET of posts for a timeline
    """
    if not session:
        print('WARN: No session for c2s_box_json')
        return None

    domain_full = get_full_domain(domain, port)
    actor = local_actor_url(http_prefix, nickname, domain_full)

    auth_header = create_basic_auth_header(nickname, password)

    profile_str = 'https://www.w3.org/ns/activitystreams'
    headers = {
        'host': domain,
        'Content-type': 'application/json',
        'Authorization': auth_header,
        'Accept': 'application/ld+json; profile="' + profile_str + '"'
    }

    # GET json
    url = actor + '/' + box_name + '?page=' + str(page_number)
    box_json = get_json(signing_priv_key_pem, session, url, headers, None,
                        debug, __version__, http_prefix, None)

    if get_json_valid(box_json) and debug:
        print('DEBUG: GET c2s_box_json success')

    return box_json


def seconds_between_published(published1: str, published2: str) -> int:
    """Returns the number of seconds between two published dates
    """
    published1_time = \
        date_from_string_format(published1, ['%Y-%m-%dT%H:%M:%S%z'])
    if not published1_time:
        print('EX: seconds_between_published unable to parse date 1 ' +
              str(published1))
        return -1
    published2_time = \
        date_from_string_format(published2, ['%Y-%m-%dT%H:%M:%S%z'])
    if not published2_time:
        print('EX: seconds_between_published unable to parse date 2 ' +
              str(published2))
        return -1
    return (published2_time - published1_time).total_seconds()


def edited_post_filename(base_dir: str, nickname: str, domain: str,
                         post_json_object: {}, debug: bool,
                         max_time_diff_seconds: int,
                         system_language: str) -> (str, {}):
    """Returns the filename of the edited post
    """
    if not has_object_dict(post_json_object):
        return '', None
    if not post_json_object.get('type'):
        return '', None
    if not post_json_object['object'].get('type'):
        return '', None
    if not post_json_object['object'].get('published'):
        return '', None
    if not post_json_object['object'].get('id'):
        return '', None
    if 'content' not in post_json_object['object']:
        return '', None
    if not post_json_object['object'].get('attributedTo'):
        return '', None
    if not get_attributed_to(post_json_object['object']['attributedTo']):
        return '', None
    actor = get_attributed_to(post_json_object['object']['attributedTo'])
    actor_filename = \
        acct_dir(base_dir, nickname, domain) + '/lastpost/' + \
        actor.replace('/', '#')
    if not os.path.isfile(actor_filename):
        return '', None
    post_id = remove_id_ending(post_json_object['object']['id'])
    lastpost_id = None
    try:
        with open(actor_filename, 'r',
                  encoding='utf-8') as fp_actor:
            lastpost_id = fp_actor.read()
    except OSError:
        print('EX: edited_post_filename unable to read ' + actor_filename)
        return '', None
    if not lastpost_id:
        return '', None
    if lastpost_id == post_id:
        return '', None
    lastpost_filename = \
        locate_post(base_dir, nickname, domain, lastpost_id, False)
    if not lastpost_filename:
        return '', None
    lastpost_json = load_json(lastpost_filename, 0)
    if not lastpost_json:
        return '', None
    if not lastpost_json.get('type'):
        return '', None
    if lastpost_json['type'] != post_json_object['type']:
        return '', None
    if not lastpost_json.get('object'):
        return '', None
    if not isinstance(lastpost_json['object'], dict):
        return '', None
    if not lastpost_json['object'].get('type'):
        return '', None
    if lastpost_json['object']['type'] != post_json_object['object']['type']:
        return '', None
    if not lastpost_json['object'].get('published'):
        return '', None
    if not lastpost_json['object'].get('id'):
        return '', None
    if 'content' not in lastpost_json['object']:
        return '', None
    if not lastpost_json['object'].get('attributedTo'):
        return '', None
    if not get_attributed_to(lastpost_json['object']['attributedTo']):
        return '', None
    time_diff_seconds = \
        seconds_between_published(lastpost_json['object']['published'],
                                  post_json_object['object']['published'])
    if time_diff_seconds > max_time_diff_seconds:
        return '', None
    if debug:
        print(post_id + ' might be an edit of ' + lastpost_id)
    lastpost_content = lastpost_json['object']['content']
    if 'contentMap' in lastpost_json['object']:
        if lastpost_json['object']['contentMap'].get(system_language):
            lastpost_content = \
                lastpost_json['object']['contentMap'][system_language]
    content = post_json_object['object']['content']
    if 'contentMap' in post_json_object['object']:
        if post_json_object['object']['contentMap'].get(system_language):
            content = \
                post_json_object['object']['contentMap'][system_language]
    if words_similarity(lastpost_content, content, 10) < 70:
        return '', None
    print(post_id + ' is an edit of ' + lastpost_id)
    return lastpost_filename, lastpost_json


def get_original_post_from_announce_url(announce_url: str, base_dir: str,
                                        nickname: str,
                                        domain: str) -> (str, str, str):
    """From the url of an announce this returns the actor, url and
    filename (if available) of the original post being announced
    """
    post_filename = locate_post(base_dir, nickname, domain, announce_url)
    if not post_filename:
        return None, None, None
    announce_post_json = load_json(post_filename, 0, 1)
    if not announce_post_json:
        return None, None, post_filename
    if not announce_post_json.get('type'):
        return None, None, post_filename
    if announce_post_json['type'] != 'Announce':
        return None, None, post_filename
    if not announce_post_json.get('object'):
        return None, None, post_filename
    if not isinstance(announce_post_json['object'], str):
        return None, None, post_filename
    actor = url = None
    # do we have the original post?
    orig_post_id = announce_post_json['object']
    orig_filename = locate_post(base_dir, nickname, domain, orig_post_id)
    if orig_filename:
        # we have the original post
        orig_post_json = load_json(orig_filename, 0, 1)
        if orig_post_json:
            if has_object_dict(orig_post_json):
                if orig_post_json['object'].get('attributedTo'):
                    attrib_field = \
                        orig_post_json['object']['attributedTo']
                    attrib = get_attributed_to(attrib_field)
                    if attrib:
                        actor = attrib
                        url = orig_post_id
                elif orig_post_json['object'].get('actor'):
                    actor = get_actor_from_post(orig_post_json)
                    url = orig_post_id
    else:
        # we don't have the original post
        if has_users_path(orig_post_id):
            # get the actor from the original post url
            orig_nick = get_nickname_from_actor(orig_post_id)
            orig_domain, _ = get_domain_from_actor(orig_post_id)
            if orig_nick and orig_domain:
                actor = \
                    orig_post_id.split('/' + orig_nick + '/')[0] + \
                    '/' + orig_nick
                url = orig_post_id

    return actor, url, orig_filename


def get_max_profile_posts(base_dir: str, nickname: str, domain: str,
                          max_recent_posts: int) -> int:
    """Returns the maximum number of posts to show on the profile screen
    """
    max_posts_filename = \
        acct_dir(base_dir, nickname, domain) + '/max_profile_posts.txt'
    max_profile_posts = 4
    if not os.path.isfile(max_posts_filename):
        return max_profile_posts
    try:
        with open(max_posts_filename, 'r', encoding='utf-8') as fp_posts:
            max_posts_str = fp_posts.read()
            if max_posts_str:
                if max_posts_str.isdigit():
                    max_profile_posts = int(max_posts_str)
    except OSError:
        print('EX: unable to read maximum profile posts ' +
              max_posts_filename)
    if max_profile_posts < 1:
        max_profile_posts = 1
    if max_profile_posts > max_recent_posts:
        max_profile_posts = max_recent_posts
    return max_profile_posts


def set_max_profile_posts(base_dir: str, nickname: str, domain: str,
                          max_recent_posts: int) -> bool:
    """Sets the maximum number of posts to show on the profile screen
    """
    max_posts_filename = \
        acct_dir(base_dir, nickname, domain) + '/max_profile_posts.txt'
    max_recent_posts_str = str(max_recent_posts)
    try:
        with open(max_posts_filename, 'w+',
                  encoding='utf-8') as fp_posts:
            fp_posts.write(max_recent_posts_str)
    except OSError:
        print('EX: unable to save maximum profile posts ' +
              max_posts_filename)
        return False
    return True