epicyon/shares.py

__filename__ = "shares.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.3.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Timeline"

import os
import re
import secrets
import time
import datetime
from random import randint
from pprint import pprint
from session import get_json
from webfinger import webfinger_handle
from auth import create_basic_auth_header
from auth import constant_time_string_check
from posts import get_person_box
from session import post_json
from session import post_image
from session import create_session
from utils import has_object_string_type
from utils import date_string_to_seconds
from utils import date_seconds_to_string
from utils import get_config_param
from utils import get_full_domain
from utils import valid_nickname
from utils import load_json
from utils import save_json
from utils import get_image_extensions
from utils import remove_domain_port
from utils import is_account_dir
from utils import acct_dir
from utils import is_float
from utils import get_category_types
from utils import get_shares_files_list
from utils import local_actor_url
from media import process_meta_data
from media import convert_image_to_low_bandwidth
from filters import is_filtered_globally
from siteactive import site_is_active
from content import get_price_from_string
from blocking import is_blocked


def _load_dfc_ids(base_dir: str, system_language: str,
                  product_type: str,
                  http_prefix: str, domain_full: str) -> {}:
    """Loads the product types ontology
    This is used to add an id to shared items
    """
    product_types_filename = \
        base_dir + '/ontology/custom' + product_type.title() + 'Types.json'
    if not os.path.isfile(product_types_filename):
        product_types_filename = \
            base_dir + '/ontology/' + product_type + 'Types.json'
    product_types = load_json(product_types_filename)
    if not product_types:
        print('Unable to load ontology: ' + product_types_filename)
        return None
    if not product_types.get('@graph'):
        print('No @graph list within ontology')
        return None
    if len(product_types['@graph']) == 0:
        print('@graph list has no contents')
        return None
    if not product_types['@graph'][0].get('rdfs:label'):
        print('@graph list entry has no rdfs:label')
        return None
    language_exists = False
    for label in product_types['@graph'][0]['rdfs:label']:
        if not label.get('@language'):
            continue
        if label['@language'] == system_language:
            language_exists = True
            break
    if not language_exists:
        print('product_types ontology does not contain the language ' +
              system_language)
        return None
    dfc_ids = {}
    for item in product_types['@graph']:
        if not item.get('@id'):
            continue
        if not item.get('rdfs:label'):
            continue
        for label in item['rdfs:label']:
            if not label.get('@language'):
                continue
            if not label.get('@value'):
                continue
            if label['@language'] == system_language:
                item_id = \
                    item['@id'].replace('http://static.datafoodconsortium.org',
                                        http_prefix + '://' + domain_full)
                dfc_ids[label['@value'].lower()] = item_id
                break
    return dfc_ids


def _get_valid_shared_item_id(actor: str, display_name: str) -> str:
    """Removes any invalid characters from the display name to
    produce an item ID
    """
    remove_chars = (' ', '\n', '\r', '#')
    for char in remove_chars:
        display_name = display_name.replace(char, '')
    remove_chars2 = ('+', '/', '\\', '?', '&')
    for char in remove_chars2:
        display_name = display_name.replace(char, '-')
    display_name = display_name.replace('.', '_')
    display_name = display_name.replace("’", "'")
    actor = actor.replace('://', '___')
    actor = actor.replace('/', '--')
    return actor + '--shareditems--' + display_name


def remove_shared_item(base_dir: str, nickname: str, domain: str,
                       item_id: str,
                       http_prefix: str, domain_full: str,
                       shares_file_type: str) -> None:
    """Removes a share for a person
    """
    shares_filename = \
        acct_dir(base_dir, nickname, domain) + '/' + shares_file_type + '.json'
    if not os.path.isfile(shares_filename):
        print('ERROR: remove shared item, missing ' +
              shares_file_type + '.json ' + shares_filename)
        return

    shares_json = load_json(shares_filename)
    if not shares_json:
        print('ERROR: remove shared item, ' +
              shares_file_type + '.json could not be loaded from ' +
              shares_filename)
        return

    if shares_json.get(item_id):
        # remove any image for the item
        item_idfile = base_dir + '/sharefiles/' + nickname + '/' + item_id
        if shares_json[item_id]['imageUrl']:
            formats = get_image_extensions()
            for ext in formats:
                if shares_json[item_id]['imageUrl'].endswith('.' + ext):
                    if os.path.isfile(item_idfile + '.' + ext):
                        try:
                            os.remove(item_idfile + '.' + ext)
                        except OSError:
                            print('EX: remove_shared_item unable to delete ' +
                                  item_idfile + '.' + ext)
        # remove the item itself
        del shares_json[item_id]
        save_json(shares_json, shares_filename)
    else:
        print('ERROR: share index "' + item_id +
              '" does not exist in ' + shares_filename)


def _add_share_duration_sec(duration: str, published: int) -> int:
    """Returns the duration for the shared item in seconds
    """
    if ' ' not in duration:
        return 0
    duration_list = duration.split(' ')
    if not duration_list[0].isdigit():
        return 0
    if 'hour' in duration_list[1]:
        return published + (int(duration_list[0]) * 60 * 60)
    if 'day' in duration_list[1]:
        return published + (int(duration_list[0]) * 60 * 60 * 24)
    if 'week' in duration_list[1]:
        return published + (int(duration_list[0]) * 60 * 60 * 24 * 7)
    if 'month' in duration_list[1]:
        return published + (int(duration_list[0]) * 60 * 60 * 24 * 30)
    if 'year' in duration_list[1]:
        return published + (int(duration_list[0]) * 60 * 60 * 24 * 365)
    return 0


def _dfc_product_type_from_category(base_dir: str,
                                    item_category: str, translate: {}) -> str:
    """Does the shared item category match a DFC product type?
    If so then return the product type.
    This will be used to select an appropriate ontology file
    such as ontology/foodTypes.json
    """
    product_types_list = get_category_types(base_dir)
    category_lower = item_category.lower()
    for product_type in product_types_list:
        if translate.get(product_type):
            if translate[product_type] in category_lower:
                return product_type
        else:
            if product_type in category_lower:
                return product_type
    return None


def _getshare_dfc_id(base_dir: str, system_language: str,
                     item_type: str, item_category: str,
                     translate: {},
                     http_prefix: str, domain_full: str,
                     dfc_ids: {} = None) -> str:
    """Attempts to obtain a DFC Id for the shared item,
    based upon product_types ontology.
    See https://github.com/datafoodconsortium/ontology
    """
    # does the category field match any prodyct type ontology
    # files in the ontology subdirectory?
    matched_product_type = \
        _dfc_product_type_from_category(base_dir, item_category, translate)
    if not matched_product_type:
        item_type = item_type.replace(' ', '_')
        item_type = item_type.replace('.', '')
        return 'epicyon#' + item_type
    if not dfc_ids:
        dfc_ids = _load_dfc_ids(base_dir, system_language,
                                matched_product_type,
                                http_prefix, domain_full)
        if not dfc_ids:
            return ''
    item_type_lower = item_type.lower()
    match_name = ''
    match_id = ''
    for name, uri in dfc_ids.items():
        if name not in item_type_lower:
            continue
        if len(name) > len(match_name):
            match_name = name
            match_id = uri
    if not match_id:
        # bag of words match
        max_matched_words = 0
        for name, uri in dfc_ids.items():
            name = name.replace('-', ' ')
            words = name.split(' ')
            score = 0
            for wrd in words:
                if wrd in item_type_lower:
                    score += 1
            if score > max_matched_words:
                max_matched_words = score
                match_id = uri
    return match_id


def _getshare_type_from_dfc_id(dfc_uri: str, dfc_ids: {}) -> str:
    """Attempts to obtain a share item type from its DFC Id,
    based upon product_types ontology.
    See https://github.com/datafoodconsortium/ontology
    """
    if dfc_uri.startswith('epicyon#'):
        item_type = dfc_uri.split('#')[1]
        item_type = item_type.replace('_', ' ')
        return item_type

    for name, uri in dfc_ids.items():
        if uri.endswith('#' + dfc_uri):
            return name
        if uri == dfc_uri:
            return name
    return None


def _indicate_new_share_available(base_dir: str, http_prefix: str,
                                  nickname: str, domain: str,
                                  domain_full: str,
                                  shares_file_type: str) -> None:
    """Indicate to each account that a new share is available
    """
    for _, dirs, _ in os.walk(base_dir + '/accounts'):
        for handle in dirs:
            if not is_account_dir(handle):
                continue
            account_dir = base_dir + '/accounts/' + handle
            if shares_file_type == 'shares':
                new_share_file = account_dir + '/.newShare'
            else:
                new_share_file = account_dir + '/.newWanted'
            if os.path.isfile(new_share_file):
                continue
            account_nickname = handle.split('@')[0]
            # does this account block you?
            if account_nickname != nickname:
                if is_blocked(base_dir, account_nickname, domain,
                              nickname, domain, None):
                    continue
            local_actor = \
                local_actor_url(http_prefix, account_nickname, domain_full)
            try:
                with open(new_share_file, 'w+') as fp_new:
                    if shares_file_type == 'shares':
                        fp_new.write(local_actor + '/tlshares')
                    else:
                        fp_new.write(local_actor + '/tlwanted')
            except OSError:
                print('EX: _indicate_new_share_available unable to write ' +
                      str(new_share_file))
        break


def add_share(base_dir: str,
              http_prefix: str, nickname: str, domain: str, port: int,
              display_name: str, summary: str, image_filename: str,
              item_qty: float, item_type: str, item_category: str,
              location: str, duration: str, debug: bool, city: str,
              price: str, currency: str,
              system_language: str, translate: {},
              shares_file_type: str, low_bandwidth: bool,
              content_license_url: str) -> None:
    """Adds a new share
    """
    if is_filtered_globally(base_dir,
                            display_name + ' ' + summary + ' ' +
                            item_type + ' ' + item_category):
        print('Shared item was filtered due to content')
        return
    shares_filename = \
        acct_dir(base_dir, nickname, domain) + '/' + shares_file_type + '.json'
    shares_json = {}
    if os.path.isfile(shares_filename):
        shares_json = load_json(shares_filename, 1, 2)

    duration = duration.lower()
    published = int(time.time())
    duration_sec = _add_share_duration_sec(duration, published)

    domain_full = get_full_domain(domain, port)
    actor = local_actor_url(http_prefix, nickname, domain_full)
    item_id = _get_valid_shared_item_id(actor, display_name)
    dfc_id = _getshare_dfc_id(base_dir, system_language,
                              item_type, item_category, translate,
                              http_prefix, domain_full)

    # has an image for this share been uploaded?
    image_url = None
    move_image = False
    if not image_filename:
        shares_image_filename = \
            acct_dir(base_dir, nickname, domain) + '/upload'
        formats = get_image_extensions()
        for ext in formats:
            if os.path.isfile(shares_image_filename + '.' + ext):
                image_filename = shares_image_filename + '.' + ext
                move_image = True

    domain_full = get_full_domain(domain, port)

    # copy or move the image for the shared item to its destination
    if image_filename:
        if os.path.isfile(image_filename):
            if not os.path.isdir(base_dir + '/sharefiles'):
                os.mkdir(base_dir + '/sharefiles')
            if not os.path.isdir(base_dir + '/sharefiles/' + nickname):
                os.mkdir(base_dir + '/sharefiles/' + nickname)
            item_idfile = base_dir + '/sharefiles/' + nickname + '/' + item_id
            formats = get_image_extensions()
            for ext in formats:
                if not image_filename.endswith('.' + ext):
                    continue
                if low_bandwidth:
                    convert_image_to_low_bandwidth(image_filename)
                process_meta_data(base_dir, nickname, domain,
                                  image_filename, item_idfile + '.' + ext,
                                  city, content_license_url)
                if move_image:
                    try:
                        os.remove(image_filename)
                    except OSError:
                        print('EX: add_share unable to delete ' +
                              str(image_filename))
                image_url = \
                    http_prefix + '://' + domain_full + \
                    '/sharefiles/' + nickname + '/' + item_id + '.' + ext

    shares_json[item_id] = {
        "displayName": display_name,
        "summary": summary,
        "imageUrl": image_url,
        "itemQty": float(item_qty),
        "dfcId": dfc_id,
        "itemType": item_type,
        "category": item_category,
        "location": location,
        "published": published,
        "expire": duration_sec,
        "itemPrice": price,
        "itemCurrency": currency
    }

    save_json(shares_json, shares_filename)

    _indicate_new_share_available(base_dir, http_prefix,
                                  nickname, domain, domain_full,
                                  shares_file_type)


def expire_shares(base_dir: str) -> None:
    """Removes expired items from shares
    """
    for _, dirs, _ in os.walk(base_dir + '/accounts'):
        for account in dirs:
            if not is_account_dir(account):
                continue
            nickname = account.split('@')[0]
            domain = account.split('@')[1]
            for shares_file_type in get_shares_files_list():
                _expire_shares_for_account(base_dir, nickname, domain,
                                           shares_file_type)
        break


def _expire_shares_for_account(base_dir: str, nickname: str, domain: str,
                               shares_file_type: str) -> None:
    """Removes expired items from shares for a particular account
    """
    handle_domain = remove_domain_port(domain)
    handle = nickname + '@' + handle_domain
    shares_filename = \
        base_dir + '/accounts/' + handle + '/' + shares_file_type + '.json'
    if not os.path.isfile(shares_filename):
        return
    shares_json = load_json(shares_filename, 1, 2)
    if not shares_json:
        return
    curr_time = int(time.time())
    delete_item_id = []
    for item_id, item in shares_json.items():
        if curr_time > item['expire']:
            delete_item_id.append(item_id)
    if not delete_item_id:
        return
    for item_id in delete_item_id:
        del shares_json[item_id]
        # remove any associated images
        item_idfile = base_dir + '/sharefiles/' + nickname + '/' + item_id
        formats = get_image_extensions()
        for ext in formats:
            if os.path.isfile(item_idfile + '.' + ext):
                try:
                    os.remove(item_idfile + '.' + ext)
                except OSError:
                    print('EX: _expire_shares_for_account unable to delete ' +
                          item_idfile + '.' + ext)
    save_json(shares_json, shares_filename)


def get_shares_feed_for_person(base_dir: str,
                               domain: str, port: int,
                               path: str, http_prefix: str,
                               shares_file_type: str,
                               shares_per_page: int) -> {}:
    """Returns the shares for an account from GET requests
    """
    if '/' + shares_file_type not in path:
        return None
    # handle page numbers
    header_only = True
    page_number = None
    if '?page=' in path:
        page_number = path.split('?page=')[1]
        if page_number == 'true':
            page_number = 1
        else:
            try:
                page_number = int(page_number)
            except BaseException:
                print('EX: get_shares_feed_for_person ' +
                      'unable to convert to int ' + str(page_number))
        path = path.split('?page=')[0]
        header_only = False

    if not path.endswith('/' + shares_file_type):
        return None
    nickname = None
    if path.startswith('/users/'):
        nickname = \
            path.replace('/users/', '', 1).replace('/' + shares_file_type, '')
    if path.startswith('/@'):
        nickname = \
            path.replace('/@', '', 1).replace('/' + shares_file_type, '')
    if not nickname:
        return None
    if not valid_nickname(domain, nickname):
        return None

    domain = get_full_domain(domain, port)

    handle_domain = remove_domain_port(domain)
    shares_filename = \
        acct_dir(base_dir, nickname, handle_domain) + '/' + \
        shares_file_type + '.json'

    if header_only:
        no_of_shares = 0
        if os.path.isfile(shares_filename):
            shares_json = load_json(shares_filename)
            if shares_json:
                no_of_shares = len(shares_json.items())
        id_str = local_actor_url(http_prefix, nickname, domain)
        shares = {
            '@context': 'https://www.w3.org/ns/activitystreams',
            'first': id_str + '/' + shares_file_type + '?page=1',
            'id': id_str + '/' + shares_file_type,
            'totalItems': str(no_of_shares),
            'type': 'OrderedCollection'
        }
        return shares

    if not page_number:
        page_number = 1

    next_page_number = int(page_number + 1)
    id_str = local_actor_url(http_prefix, nickname, domain)
    shares = {
        '@context': 'https://www.w3.org/ns/activitystreams',
        'id': id_str + '/' + shares_file_type + '?page=' + str(page_number),
        'orderedItems': [],
        'partOf': id_str + '/' + shares_file_type,
        'totalItems': 0,
        'type': 'OrderedCollectionPage'
    }

    if not os.path.isfile(shares_filename):
        return shares
    curr_page = 1
    page_ctr = 0
    total_ctr = 0

    shares_json = load_json(shares_filename)
    if shares_json:
        for item_id, item in shares_json.items():
            page_ctr += 1
            total_ctr += 1
            if curr_page == page_number:
                item['shareId'] = item_id
                shares['orderedItems'].append(item)
            if page_ctr >= shares_per_page:
                page_ctr = 0
                curr_page += 1
    shares['totalItems'] = total_ctr
    last_page = int(total_ctr / shares_per_page)
    if last_page < 1:
        last_page = 1
    if next_page_number > last_page:
        shares['next'] = \
            local_actor_url(http_prefix, nickname, domain) + \
            '/' + shares_file_type + '?page=' + str(last_page)
    return shares


def send_share_via_server(base_dir, session,
                          from_nickname: str, password: str,
                          from_domain: str, fromPort: int,
                          http_prefix: str, display_name: str,
                          summary: str, image_filename: str,
                          item_qty: float, item_type: str, item_category: str,
                          location: str, duration: str,
                          cached_webfingers: {}, person_cache: {},
                          debug: bool, project_version: str,
                          itemPrice: str, item_currency: str,
                          signing_priv_key_pem: str) -> {}:
    """Creates an item share via c2s
    """
    if not session:
        print('WARN: No session for send_share_via_server')
        return 6

    # convert $4.23 to 4.23 USD
    new_item_price, new_item_currency = get_price_from_string(itemPrice)
    if new_item_price != itemPrice:
        itemPrice = new_item_price
        if not item_currency:
            if new_item_currency != item_currency:
                item_currency = new_item_currency

    from_domain_full = get_full_domain(from_domain, fromPort)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = actor + '/followers'

    new_share_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Add',
        'actor': actor,
        'target': actor + '/shares',
        'object': {
            "type": "Offer",
            "displayName": display_name,
            "summary": summary,
            "itemQty": float(item_qty),
            "itemType": item_type,
            "category": item_category,
            "location": location,
            "duration": duration,
            "itemPrice": itemPrice,
            "itemCurrency": item_currency,
            'to': [to_url],
            'cc': [cc_url]
        },
        'to': [to_url],
        'cc': [cc_url]
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = \
        webfinger_handle(session, handle, http_prefix,
                         cached_webfingers,
                         from_domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: share webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: share webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     display_name, _) = get_person_box(signing_priv_key_pem,
                                       origin_domain,
                                       base_dir, session, wf_request,
                                       person_cache, project_version,
                                       http_prefix, from_nickname,
                                       from_domain, post_to_box,
                                       83653)

    if not inbox_url:
        if debug:
            print('DEBUG: share no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: share no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    if image_filename:
        headers = {
            'host': from_domain,
            'Authorization': auth_header
        }
        inbox_url_str = inbox_url.replace('/' + post_to_box, '/shares')
        post_result = \
            post_image(session, image_filename, [], inbox_url_str,
                       headers, http_prefix, from_domain_full)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = \
        post_json(http_prefix, from_domain_full,
                  session, new_share_json, [], inbox_url, headers, 30, True)
    if not post_result:
        if debug:
            print('DEBUG: POST share failed for c2s to ' + inbox_url)
#        return 5

    if debug:
        print('DEBUG: c2s POST share item success')

    return new_share_json


def send_undo_share_via_server(base_dir: str, session,
                               from_nickname: str, password: str,
                               from_domain: str, fromPort: int,
                               http_prefix: str, display_name: str,
                               cached_webfingers: {}, person_cache: {},
                               debug: bool, project_version: str,
                               signing_priv_key_pem: str) -> {}:
    """Undoes a share via c2s
    """
    if not session:
        print('WARN: No session for send_undo_share_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, fromPort)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = actor + '/followers'

    undo_share_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Remove',
        'actor': actor,
        'target': actor + '/shares',
        'object': {
            "type": "Offer",
            "displayName": display_name,
            'to': [to_url],
            'cc': [cc_url]
        },
        'to': [to_url],
        'cc': [cc_url]
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         from_domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: unshare webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: unshare webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     display_name, _) = get_person_box(signing_priv_key_pem,
                                       origin_domain,
                                       base_dir, session, wf_request,
                                       person_cache, project_version,
                                       http_prefix, from_nickname,
                                       from_domain, post_to_box,
                                       12663)

    if not inbox_url:
        if debug:
            print('DEBUG: unshare no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: unshare no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = \
        post_json(http_prefix, from_domain_full,
                  session, undo_share_json, [], inbox_url,
                  headers, 30, True)
    if not post_result:
        if debug:
            print('DEBUG: POST unshare failed for c2s to ' + inbox_url)
#        return 5

    if debug:
        print('DEBUG: c2s POST unshare success')

    return undo_share_json


def send_wanted_via_server(base_dir, session,
                           from_nickname: str, password: str,
                           from_domain: str, fromPort: int,
                           http_prefix: str, display_name: str,
                           summary: str, image_filename: str,
                           item_qty: float, item_type: str, item_category: str,
                           location: str, duration: str,
                           cached_webfingers: {}, person_cache: {},
                           debug: bool, project_version: str,
                           itemMaxPrice: str, item_currency: str,
                           signing_priv_key_pem: str) -> {}:
    """Creates a wanted item via c2s
    """
    if not session:
        print('WARN: No session for send_wanted_via_server')
        return 6

    # convert $4.23 to 4.23 USD
    new_item_max_price, new_item_currency = get_price_from_string(itemMaxPrice)
    if new_item_max_price != itemMaxPrice:
        itemMaxPrice = new_item_max_price
        if not item_currency:
            if new_item_currency != item_currency:
                item_currency = new_item_currency

    from_domain_full = get_full_domain(from_domain, fromPort)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = actor + '/followers'

    new_share_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Add',
        'actor': actor,
        'target': actor + '/wanted',
        'object': {
            "type": "Offer",
            "displayName": display_name,
            "summary": summary,
            "itemQty": float(item_qty),
            "itemType": item_type,
            "category": item_category,
            "location": location,
            "duration": duration,
            "itemPrice": itemMaxPrice,
            "itemCurrency": item_currency,
            'to': [to_url],
            'cc': [cc_url]
        },
        'to': [to_url],
        'cc': [cc_url]
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = \
        webfinger_handle(session, handle, http_prefix,
                         cached_webfingers,
                         from_domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: share webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: wanted webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     display_name, _) = get_person_box(signing_priv_key_pem,
                                       origin_domain,
                                       base_dir, session, wf_request,
                                       person_cache, project_version,
                                       http_prefix, from_nickname,
                                       from_domain, post_to_box,
                                       23653)

    if not inbox_url:
        if debug:
            print('DEBUG: wanted no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: wanted no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    if image_filename:
        headers = {
            'host': from_domain,
            'Authorization': auth_header
        }
        inbox_url_str = inbox_url.replace('/' + post_to_box, '/wanted')
        post_result = \
            post_image(session, image_filename, [], inbox_url_str,
                       headers, http_prefix, from_domain_full)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = \
        post_json(http_prefix, from_domain_full,
                  session, new_share_json, [], inbox_url, headers, 30, True)
    if not post_result:
        if debug:
            print('DEBUG: POST wanted failed for c2s to ' + inbox_url)
#        return 5

    if debug:
        print('DEBUG: c2s POST wanted item success')

    return new_share_json


def send_undo_wanted_via_server(base_dir: str, session,
                                from_nickname: str, password: str,
                                from_domain: str, fromPort: int,
                                http_prefix: str, display_name: str,
                                cached_webfingers: {}, person_cache: {},
                                debug: bool, project_version: str,
                                signing_priv_key_pem: str) -> {}:
    """Undoes a wanted item via c2s
    """
    if not session:
        print('WARN: No session for send_undo_wanted_via_server')
        return 6

    from_domain_full = get_full_domain(from_domain, fromPort)

    actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
    to_url = 'https://www.w3.org/ns/activitystreams#Public'
    cc_url = actor + '/followers'

    undo_share_json = {
        "@context": "https://www.w3.org/ns/activitystreams",
        'type': 'Remove',
        'actor': actor,
        'target': actor + '/wanted',
        'object': {
            "type": "Offer",
            "displayName": display_name,
            'to': [to_url],
            'cc': [cc_url]
        },
        'to': [to_url],
        'cc': [cc_url]
    }

    handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname

    # lookup the inbox for the To handle
    wf_request = \
        webfinger_handle(session, handle, http_prefix, cached_webfingers,
                         from_domain, project_version, debug, False,
                         signing_priv_key_pem)
    if not wf_request:
        if debug:
            print('DEBUG: unwant webfinger failed for ' + handle)
        return 1
    if not isinstance(wf_request, dict):
        print('WARN: unwant webfinger for ' + handle +
              ' did not return a dict. ' + str(wf_request))
        return 1

    post_to_box = 'outbox'

    # get the actor inbox for the To handle
    origin_domain = from_domain
    (inbox_url, _, _, from_person_id, _, _,
     display_name, _) = get_person_box(signing_priv_key_pem,
                                       origin_domain,
                                       base_dir, session, wf_request,
                                       person_cache, project_version,
                                       http_prefix, from_nickname,
                                       from_domain, post_to_box,
                                       12693)

    if not inbox_url:
        if debug:
            print('DEBUG: unwant no ' + post_to_box +
                  ' was found for ' + handle)
        return 3
    if not from_person_id:
        if debug:
            print('DEBUG: unwant no actor was found for ' + handle)
        return 4

    auth_header = create_basic_auth_header(from_nickname, password)

    headers = {
        'host': from_domain,
        'Content-type': 'application/json',
        'Authorization': auth_header
    }
    post_result = \
        post_json(http_prefix, from_domain_full,
                  session, undo_share_json, [], inbox_url,
                  headers, 30, True)
    if not post_result:
        if debug:
            print('DEBUG: POST unwant failed for c2s to ' + inbox_url)
#        return 5

    if debug:
        print('DEBUG: c2s POST unwant success')

    return undo_share_json


def get_shared_items_catalog_via_server(base_dir, session,
                                        nickname: str, password: str,
                                        domain: str, port: int,
                                        http_prefix: str, debug: bool,
                                        signing_priv_key_pem: str) -> {}:
    """Returns the shared items catalog via c2s
    """
    if not session:
        print('WARN: No session for get_shared_items_catalog_via_server')
        return 6

    auth_header = create_basic_auth_header(nickname, password)

    headers = {
        'host': domain,
        'Content-type': 'application/json',
        'Authorization': auth_header,
        'Accept': 'application/json'
    }
    domain_full = get_full_domain(domain, port)
    url = local_actor_url(http_prefix, nickname, domain_full) + '/catalog'
    if debug:
        print('Shared items catalog request to: ' + url)
    catalog_json = get_json(signing_priv_key_pem, session, url, headers, None,
                            debug, __version__, http_prefix, None)
    if not catalog_json:
        if debug:
            print('DEBUG: GET shared items catalog failed for c2s to ' + url)
#        return 5

    if debug:
        print('DEBUG: c2s GET shared items catalog success')

    return catalog_json


def outbox_share_upload(base_dir: str, http_prefix: str,
                        nickname: str, domain: str, port: int,
                        message_json: {}, debug: bool, city: str,
                        system_language: str, translate: {},
                        low_bandwidth: bool,
                        content_license_url: str) -> None:
    """ When a shared item is received by the outbox from c2s
    """
    if not message_json.get('type'):
        return
    if not message_json['type'] == 'Add':
        return
    if not has_object_string_type(message_json, debug):
        return
    if not message_json['object']['type'] == 'Offer':
        if debug:
            print('DEBUG: not an Offer activity')
        return
    if not message_json['object'].get('displayName'):
        if debug:
            print('DEBUG: displayName missing from Offer')
        return
    if not message_json['object'].get('summary'):
        if debug:
            print('DEBUG: summary missing from Offer')
        return
    if not message_json['object'].get('itemQty'):
        if debug:
            print('DEBUG: itemQty missing from Offer')
        return
    if not message_json['object'].get('itemType'):
        if debug:
            print('DEBUG: itemType missing from Offer')
        return
    if not message_json['object'].get('category'):
        if debug:
            print('DEBUG: category missing from Offer')
        return
    if not message_json['object'].get('duration'):
        if debug:
            print('DEBUG: duration missing from Offer')
        return
    item_qty = float(message_json['object']['itemQty'])
    location = ''
    if message_json['object'].get('location'):
        location = message_json['object']['location']
    image_filename = None
    if message_json['object'].get('image_filename'):
        image_filename = message_json['object']['image_filename']
    if debug:
        print('Adding shared item')
        pprint(message_json)

    add_share(base_dir,
              http_prefix, nickname, domain, port,
              message_json['object']['displayName'],
              message_json['object']['summary'],
              image_filename,
              item_qty,
              message_json['object']['itemType'],
              message_json['object']['category'],
              location,
              message_json['object']['duration'],
              debug, city,
              message_json['object']['itemPrice'],
              message_json['object']['itemCurrency'],
              system_language, translate, 'shares',
              low_bandwidth, content_license_url)
    if debug:
        print('DEBUG: shared item received via c2s')


def outbox_undo_share_upload(base_dir: str, http_prefix: str,
                             nickname: str, domain: str, port: int,
                             message_json: {}, debug: bool) -> None:
    """ When a shared item is removed via c2s
    """
    if not message_json.get('type'):
        return
    if not message_json['type'] == 'Remove':
        return
    if not has_object_string_type(message_json, debug):
        return
    if not message_json['object']['type'] == 'Offer':
        if debug:
            print('DEBUG: not an Offer activity')
        return
    if not message_json['object'].get('displayName'):
        if debug:
            print('DEBUG: displayName missing from Offer')
        return
    domain_full = get_full_domain(domain, port)
    remove_shared_item(base_dir, nickname, domain,
                       message_json['object']['displayName'],
                       http_prefix, domain_full, 'shares')
    if debug:
        print('DEBUG: shared item removed via c2s')


def _shares_catalog_params(path: str) -> (bool, float, float, str):
    """Returns parameters when accessing the shares catalog
    """
    today = False
    min_price = 0
    max_price = 9999999
    match_pattern = None
    if '?' not in path:
        return today, min_price, max_price, match_pattern
    args = path.split('?', 1)[1]
    arg_list = args.split(';')
    for arg in arg_list:
        if '=' not in arg:
            continue
        key = arg.split('=')[0].lower()
        value = arg.split('=')[1]
        if key == 'today':
            value = value.lower()
            if 't' in value or 'y' in value or '1' in value:
                today = True
        elif key.startswith('min'):
            if is_float(value):
                min_price = float(value)
        elif key.startswith('max'):
            if is_float(value):
                max_price = float(value)
        elif key.startswith('match'):
            match_pattern = value
    return today, min_price, max_price, match_pattern


def shares_catalog_account_endpoint(base_dir: str, http_prefix: str,
                                    nickname: str, domain: str,
                                    domain_full: str,
                                    path: str, debug: bool,
                                    shares_file_type: str) -> {}:
    """Returns the endpoint for the shares catalog of a particular account
    See https://github.com/datafoodconsortium/ontology
    Also the subdirectory ontology/DFC
    """
    today, min_price, max_price, match_pattern = _shares_catalog_params(path)
    dfc_url = \
        http_prefix + '://' + domain_full + '/ontologies/DFC_FullModel.owl#'
    dfc_pt_url = \
        http_prefix + '://' + domain_full + \
        '/ontologies/DFC_ProductGlossary.rdf#'
    owner = local_actor_url(http_prefix, nickname, domain_full)
    if shares_file_type == 'shares':
        dfc_instance_id = owner + '/catalog'
    else:
        dfc_instance_id = owner + '/wantedItems'
    endpoint = {
        "@context": {
            "DFC": dfc_url,
            "dfc-pt": dfc_pt_url,
            "@base": "http://maPlateformeNationale"
        },
        "@id": dfc_instance_id,
        "@type": "DFC:Entreprise",
        "DFC:supplies": []
    }

    curr_date = datetime.datetime.utcnow()
    curr_date_str = curr_date.strftime("%Y-%m-%d")

    shares_filename = \
        acct_dir(base_dir, nickname, domain) + '/' + shares_file_type + '.json'
    if not os.path.isfile(shares_filename):
        if debug:
            print(shares_file_type + '.json file not found: ' +
                  shares_filename)
        return endpoint
    shares_json = load_json(shares_filename, 1, 2)
    if not shares_json:
        if debug:
            print('Unable to load json for ' + shares_filename)
        return endpoint

    for item_id, item in shares_json.items():
        if not item.get('dfcId'):
            if debug:
                print('Item does not have dfcId: ' + item_id)
            continue
        if '#' not in item['dfcId']:
            continue
        if today:
            if not item['published'].startswith(curr_date_str):
                continue
        if min_price is not None:
            if float(item['itemPrice']) < min_price:
                continue
        if max_price is not None:
            if float(item['itemPrice']) > max_price:
                continue
        description = item['displayName'] + ': ' + item['summary']
        if match_pattern:
            if not re.match(match_pattern, description):
                continue

        expire_date = datetime.datetime.fromtimestamp(item['expire'])
        expire_date_str = expire_date.strftime("%Y-%m-%dT%H:%M:%SZ")

        share_id = _get_valid_shared_item_id(owner, item['displayName'])
        if item['dfcId'].startswith('epicyon#'):
            dfc_id = "epicyon:" + item['dfcId'].split('#')[1]
        else:
            dfc_id = "dfc-pt:" + item['dfcId'].split('#')[1]
        price_str = item['itemPrice'] + ' ' + item['itemCurrency']
        catalog_item = {
            "@id": share_id,
            "@type": "DFC:SuppliedProduct",
            "DFC:hasType": dfc_id,
            "DFC:startDate": item['published'],
            "DFC:expiryDate": expire_date_str,
            "DFC:quantity": float(item['itemQty']),
            "DFC:price": price_str,
            "DFC:Image": item['imageUrl'],
            "DFC:description": description
        }
        endpoint['DFC:supplies'].append(catalog_item)

    return endpoint


def shares_catalog_endpoint(base_dir: str, http_prefix: str,
                            domain_full: str,
                            path: str, shares_file_type: str) -> {}:
    """Returns the endpoint for the shares catalog for the instance
    See https://github.com/datafoodconsortium/ontology
    Also the subdirectory ontology/DFC
    """
    today, min_price, max_price, match_pattern = _shares_catalog_params(path)
    dfc_url = \
        http_prefix + '://' + domain_full + '/ontologies/DFC_FullModel.owl#'
    dfc_pt_url = \
        http_prefix + '://' + domain_full + \
        '/ontologies/DFC_ProductGlossary.rdf#'
    dfc_instance_id = http_prefix + '://' + domain_full + '/catalog'
    endpoint = {
        "@context": {
            "DFC": dfc_url,
            "dfc-pt": dfc_pt_url,
            "@base": "http://maPlateformeNationale"
        },
        "@id": dfc_instance_id,
        "@type": "DFC:Entreprise",
        "DFC:supplies": []
    }

    curr_date = datetime.datetime.utcnow()
    curr_date_str = curr_date.strftime("%Y-%m-%d")

    for _, dirs, _ in os.walk(base_dir + '/accounts'):
        for acct in dirs:
            if not is_account_dir(acct):
                continue
            nickname = acct.split('@')[0]
            domain = acct.split('@')[1]
            owner = local_actor_url(http_prefix, nickname, domain_full)

            shares_filename = \
                acct_dir(base_dir, nickname, domain) + '/' + \
                shares_file_type + '.json'
            if not os.path.isfile(shares_filename):
                continue
            print('Test 78363 ' + shares_filename)
            shares_json = load_json(shares_filename, 1, 2)
            if not shares_json:
                continue

            for _, item in shares_json.items():
                if not item.get('dfcId'):
                    continue
                if '#' not in item['dfcId']:
                    continue
                if today:
                    if not item['published'].startswith(curr_date_str):
                        continue
                if min_price is not None:
                    if float(item['itemPrice']) < min_price:
                        continue
                if max_price is not None:
                    if float(item['itemPrice']) > max_price:
                        continue
                description = item['displayName'] + ': ' + item['summary']
                if match_pattern:
                    if not re.match(match_pattern, description):
                        continue

                start_date_str = date_seconds_to_string(item['published'])
                expire_date_str = date_seconds_to_string(item['expire'])
                share_id = \
                    _get_valid_shared_item_id(owner, item['displayName'])
                if item['dfcId'].startswith('epicyon#'):
                    dfc_id = "epicyon:" + item['dfcId'].split('#')[1]
                else:
                    dfc_id = "dfc-pt:" + item['dfcId'].split('#')[1]
                price_str = item['itemPrice'] + ' ' + item['itemCurrency']
                catalog_item = {
                    "@id": share_id,
                    "@type": "DFC:SuppliedProduct",
                    "DFC:hasType": dfc_id,
                    "DFC:startDate": start_date_str,
                    "DFC:expiryDate": expire_date_str,
                    "DFC:quantity": float(item['itemQty']),
                    "DFC:price": price_str,
                    "DFC:Image": item['imageUrl'],
                    "DFC:description": description
                }
                endpoint['DFC:supplies'].append(catalog_item)

    return endpoint


def shares_catalog_csv_endpoint(base_dir: str, http_prefix: str,
                                domain_full: str,
                                path: str, shares_file_type: str) -> str:
    """Returns a CSV version of the shares catalog
    """
    catalog_json = \
        shares_catalog_endpoint(base_dir, http_prefix, domain_full, path,
                                shares_file_type)
    if not catalog_json:
        return ''
    if not catalog_json.get('DFC:supplies'):
        return ''
    csv_str = \
        'id,type,hasType,startDate,expiryDate,' + \
        'quantity,price,currency,Image,description,\n'
    for item in catalog_json['DFC:supplies']:
        csv_str += '"' + item['@id'] + '",'
        csv_str += '"' + item['@type'] + '",'
        csv_str += '"' + item['DFC:hasType'] + '",'
        csv_str += '"' + item['DFC:startDate'] + '",'
        csv_str += '"' + item['DFC:expiryDate'] + '",'
        csv_str += str(item['DFC:quantity']) + ','
        csv_str += item['DFC:price'].split(' ')[0] + ','
        csv_str += '"' + item['DFC:price'].split(' ')[1] + '",'
        if item.get('DFC:Image'):
            csv_str += '"' + item['DFC:Image'] + '",'
        description = item['DFC:description'].replace('"', "'")
        csv_str += '"' + description + '",\n'
    return csv_str


def generate_shared_item_federation_tokens(shared_items_federated_domains: [],
                                           base_dir: str) -> {}:
    """Generates tokens for shared item federated domains
    """
    if not shared_items_federated_domains:
        return {}

    tokens_json = {}
    if base_dir:
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        if os.path.isfile(tokens_filename):
            tokens_json = load_json(tokens_filename, 1, 2)
            if tokens_json is None:
                tokens_json = {}

    tokens_added = False
    for domain_full in shared_items_federated_domains:
        if not tokens_json.get(domain_full):
            tokens_json[domain_full] = ''
            tokens_added = True

    if not tokens_added:
        return tokens_json
    if base_dir:
        save_json(tokens_json, tokens_filename)
    return tokens_json


def update_shared_item_federation_token(base_dir: str,
                                        token_domain_full: str, new_token: str,
                                        debug: bool,
                                        tokens_json: {} = None) -> {}:
    """Updates an individual token for shared item federation
    """
    if debug:
        print('Updating shared items token for ' + token_domain_full)
    if not tokens_json:
        tokens_json = {}
    if base_dir:
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        if os.path.isfile(tokens_filename):
            if debug:
                print('Update loading tokens for ' + token_domain_full)
            tokens_json = load_json(tokens_filename, 1, 2)
            if tokens_json is None:
                tokens_json = {}
    update_required = False
    if tokens_json.get(token_domain_full):
        if tokens_json[token_domain_full] != new_token:
            update_required = True
    else:
        update_required = True
    if update_required:
        tokens_json[token_domain_full] = new_token
        if base_dir:
            save_json(tokens_json, tokens_filename)
    return tokens_json


def merge_shared_item_tokens(base_dir: str, domain_full: str,
                             new_shared_items_federated_domains: [],
                             tokens_json: {}) -> {}:
    """When the shared item federation domains list has changed, update
    the tokens dict accordingly
    """
    removals = []
    changed = False
    for token_domain_full, _ in tokens_json.items():
        if domain_full:
            if token_domain_full.startswith(domain_full):
                continue
        if token_domain_full not in new_shared_items_federated_domains:
            removals.append(token_domain_full)
    # remove domains no longer in the federation list
    for token_domain_full in removals:
        del tokens_json[token_domain_full]
        changed = True
    # add new domains from the federation list
    for token_domain_full in new_shared_items_federated_domains:
        if token_domain_full not in tokens_json:
            tokens_json[token_domain_full] = ''
            changed = True
    if base_dir and changed:
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        save_json(tokens_json, tokens_filename)
    return tokens_json


def create_shared_item_federation_token(base_dir: str,
                                        token_domain_full: str,
                                        force: bool,
                                        tokens_json: {} = None) -> {}:
    """Updates an individual token for shared item federation
    """
    if not tokens_json:
        tokens_json = {}
    if base_dir:
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        if os.path.isfile(tokens_filename):
            tokens_json = load_json(tokens_filename, 1, 2)
            if tokens_json is None:
                tokens_json = {}
    if force or not tokens_json.get(token_domain_full):
        tokens_json[token_domain_full] = secrets.token_urlsafe(64)
        if base_dir:
            save_json(tokens_json, tokens_filename)
    return tokens_json


def authorize_shared_items(shared_items_federated_domains: [],
                           base_dir: str,
                           origin_domain_full: str,
                           calling_domainFull: str,
                           auth_header: str,
                           debug: bool,
                           tokens_json: {} = None) -> bool:
    """HTTP simple token check for shared item federation
    """
    if not shared_items_federated_domains:
        # no shared item federation
        return False
    if origin_domain_full not in shared_items_federated_domains:
        if debug:
            print(origin_domain_full +
                  ' is not in the shared items federation list ' +
                  str(shared_items_federated_domains))
        return False
    if 'Basic ' in auth_header:
        if debug:
            print('DEBUG: shared item federation should not use basic auth')
        return False
    provided_token = auth_header.replace('\n', '').replace('\r', '').strip()
    if not provided_token:
        if debug:
            print('DEBUG: shared item federation token is empty')
        return False
    if len(provided_token) < 60:
        if debug:
            print('DEBUG: shared item federation token is too small ' +
                  provided_token)
        return False
    if not tokens_json:
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        if not os.path.isfile(tokens_filename):
            if debug:
                print('DEBUG: shared item federation tokens file missing ' +
                      tokens_filename)
            return False
        tokens_json = load_json(tokens_filename, 1, 2)
    if not tokens_json:
        return False
    if not tokens_json.get(calling_domainFull):
        if debug:
            print('DEBUG: shared item federation token ' +
                  'check failed for ' + calling_domainFull)
        return False
    if not constant_time_string_check(tokens_json[calling_domainFull],
                                      provided_token):
        if debug:
            print('DEBUG: shared item federation token ' +
                  'mismatch for ' + calling_domainFull)
        return False
    return True


def _update_federated_shares_cache(session, shared_items_federated_domains: [],
                                   base_dir: str, domain_full: str,
                                   http_prefix: str,
                                   tokens_json: {}, debug: bool,
                                   system_language: str,
                                   shares_file_type: str) -> None:
    """Updates the cache of federated shares for the instance.
    This enables shared items to be available even when other instances
    might not be online
    """
    # create directories where catalogs will be stored
    cache_dir = base_dir + '/cache'
    if not os.path.isdir(cache_dir):
        os.mkdir(cache_dir)
    if shares_file_type == 'shares':
        catalogs_dir = cache_dir + '/catalogs'
    else:
        catalogs_dir = cache_dir + '/wantedItems'
    if not os.path.isdir(catalogs_dir):
        os.mkdir(catalogs_dir)

    as_header = {
        "Accept": "application/ld+json",
        "Origin": domain_full
    }
    for federated_domain_full in shared_items_federated_domains:
        # NOTE: federatedDomain does not have a port extension,
        # so may not work in some situations
        if federated_domain_full.startswith(domain_full):
            # only download from instances other than this one
            continue
        if not tokens_json.get(federated_domain_full):
            # token has been obtained for the other domain
            continue
        if not site_is_active(http_prefix + '://' + federated_domain_full, 10):
            continue
        if shares_file_type == 'shares':
            url = http_prefix + '://' + federated_domain_full + '/catalog'
        else:
            url = http_prefix + '://' + federated_domain_full + '/wantedItems'
        as_header['Authorization'] = tokens_json[federated_domain_full]
        catalog_json = get_json(session, url, as_header, None,
                                debug, __version__, http_prefix, None)
        if not catalog_json:
            print('WARN: failed to download shared items catalog for ' +
                  federated_domain_full)
            continue
        catalog_filename = catalogs_dir + '/' + federated_domain_full + '.json'
        if save_json(catalog_json, catalog_filename):
            print('Downloaded shared items catalog for ' +
                  federated_domain_full)
            shares_json = _dfc_to_shares_format(catalog_json,
                                                base_dir, system_language,
                                                http_prefix, domain_full)
            if shares_json:
                shares_filename = \
                    catalogs_dir + '/' + federated_domain_full + '.' + \
                    shares_file_type + '.json'
                save_json(shares_json, shares_filename)
                print('Converted shares catalog for ' + federated_domain_full)
        else:
            time.sleep(2)


def run_federated_shares_watchdog(project_version: str, httpd) -> None:
    """This tries to keep the federated shares update thread
    running even if it dies
    """
    print('THREAD: Starting federated shares watchdog')
    federated_shares_original = \
        httpd.thrPostSchedule.clone(run_federated_shares_daemon)
    httpd.thrFederatedSharesDaemon.start()
    while True:
        time.sleep(55)
        if httpd.thrFederatedSharesDaemon.is_alive():
            continue
        httpd.thrFederatedSharesDaemon.kill()
        print('THREAD: restarting federated shares watchdog')
        httpd.thrFederatedSharesDaemon = \
            federated_shares_original.clone(run_federated_shares_daemon)
        httpd.thrFederatedSharesDaemon.start()
        print('Restarting federated shares daemon...')


def _generate_next_shares_token_update(base_dir: str,
                                       min_days: int, max_days: int) -> None:
    """Creates a file containing the next date when the shared items token
    for this instance will be updated
    """
    token_update_dir = base_dir + '/accounts'
    if not os.path.isdir(base_dir):
        os.mkdir(base_dir)
    if not os.path.isdir(token_update_dir):
        os.mkdir(token_update_dir)
    token_update_filename = token_update_dir + '/.tokenUpdate'
    next_update_sec = None
    if os.path.isfile(token_update_filename):
        with open(token_update_filename, 'r') as fp_tok:
            next_update_str = fp_tok.read()
            if next_update_str:
                if next_update_str.isdigit():
                    next_update_sec = int(next_update_str)
    curr_time = int(time.time())
    updated = False
    if next_update_sec:
        if curr_time > next_update_sec:
            next_update_days = randint(min_days, max_days)
            next_update_interval = int(60 * 60 * 24 * next_update_days)
            next_update_sec += next_update_interval
            updated = True
    else:
        next_update_days = randint(min_days, max_days)
        next_update_interval = int(60 * 60 * 24 * next_update_days)
        next_update_sec = curr_time + next_update_interval
        updated = True
    if updated:
        with open(token_update_filename, 'w+') as fp_tok:
            fp_tok.write(str(next_update_sec))


def _regenerate_shares_token(base_dir: str, domain_full: str,
                             min_days: int, max_days: int, httpd) -> None:
    """Occasionally the shared items token for your instance is updated.
    Scenario:
      - You share items with $FriendlyInstance
      - Some time later under new management
        $FriendlyInstance becomes $HostileInstance
      - You block $HostileInstance and remove them from your
        federated shares domains list
      - $HostileInstance still knows your shared items token,
        and can still have access to your shared items if it presents a
        spoofed Origin header together with the token
    By rotating the token occasionally $HostileInstance will eventually
    lose access to your federated shares. If other instances within your
    federated shares list of domains continue to follow and communicate
    then they will receive the new token automatically
    """
    token_update_filename = base_dir + '/accounts/.tokenUpdate'
    if not os.path.isfile(token_update_filename):
        return
    next_update_sec = None
    with open(token_update_filename, 'r') as fp_tok:
        next_update_str = fp_tok.read()
        if next_update_str:
            if next_update_str.isdigit():
                next_update_sec = int(next_update_str)
    if not next_update_sec:
        return
    curr_time = int(time.time())
    if curr_time <= next_update_sec:
        return
    create_shared_item_federation_token(base_dir, domain_full, True, None)
    _generate_next_shares_token_update(base_dir, min_days, max_days)
    # update the tokens used within the daemon
    shared_fed_domains = httpd.shared_items_federated_domains
    httpd.shared_item_federation_tokens = \
        generate_shared_item_federation_tokens(shared_fed_domains,
                                               base_dir)


def run_federated_shares_daemon(base_dir: str, httpd, http_prefix: str,
                                domain_full: str, proxy_type: str, debug: bool,
                                system_language: str) -> None:
    """Runs the daemon used to update federated shared items
    """
    seconds_per_hour = 60 * 60
    file_check_interval_sec = 120
    time.sleep(60)
    # the token for this instance will be changed every 7-14 days
    min_days = 7
    max_days = 14
    _generate_next_shares_token_update(base_dir, min_days, max_days)
    while True:
        shared_items_federated_domains_str = \
            get_config_param(base_dir, 'sharedItemsFederatedDomains')
        if not shared_items_federated_domains_str:
            time.sleep(file_check_interval_sec)
            continue

        # occasionally change the federated shared items token
        # for this instance
        _regenerate_shares_token(base_dir, domain_full,
                                 min_days, max_days, httpd)

        # get a list of the domains within the shared items federation
        shared_items_federated_domains = []
        fed_domains_list = \
            shared_items_federated_domains_str.split(',')
        for shared_fed_domain in fed_domains_list:
            shared_items_federated_domains.append(shared_fed_domain.strip())
        if not shared_items_federated_domains:
            time.sleep(file_check_interval_sec)
            continue

        # load the tokens
        tokens_filename = \
            base_dir + '/accounts/sharedItemsFederationTokens.json'
        if not os.path.isfile(tokens_filename):
            time.sleep(file_check_interval_sec)
            continue
        tokens_json = load_json(tokens_filename, 1, 2)
        if not tokens_json:
            time.sleep(file_check_interval_sec)
            continue

        session = create_session(proxy_type)
        for shares_file_type in get_shares_files_list():
            _update_federated_shares_cache(session,
                                           shared_items_federated_domains,
                                           base_dir, domain_full, http_prefix,
                                           tokens_json, debug, system_language,
                                           shares_file_type)
        time.sleep(seconds_per_hour * 6)


def _dfc_to_shares_format(catalog_json: {},
                          base_dir: str, system_language: str,
                          http_prefix: str, domain_full: str) -> {}:
    """Converts DFC format into the internal formal used to store shared items.
    This simplifies subsequent search and display
    """
    if not catalog_json.get('DFC:supplies'):
        return {}
    shares_json = {}

    dfc_ids = {}
    product_types_list = get_category_types(base_dir)
    for product_type in product_types_list:
        dfc_ids[product_type] = \
            _load_dfc_ids(base_dir, system_language, product_type,
                          http_prefix, domain_full)

    curr_time = int(time.time())
    for item in catalog_json['DFC:supplies']:
        if not item.get('@id') or \
           not item.get('@type') or \
           not item.get('DFC:hasType') or \
           not item.get('DFC:startDate') or \
           not item.get('DFC:expiryDate') or \
           not item.get('DFC:quantity') or \
           not item.get('DFC:price') or \
           not item.get('DFC:description'):
            continue

        if ' ' not in item['DFC:price']:
            continue
        if ':' not in item['DFC:description']:
            continue
        if ':' not in item['DFC:hasType']:
            continue

        start_time_sec = date_string_to_seconds(item['DFC:startDate'])
        if not start_time_sec:
            continue
        expiry_time_sec = date_string_to_seconds(item['DFC:expiryDate'])
        if not expiry_time_sec:
            continue
        if expiry_time_sec < curr_time:
            # has expired
            continue

        if item['DFC:hasType'].startswith('epicyon:'):
            item_type = item['DFC:hasType'].split(':')[1]
            item_type = item_type.replace('_', ' ')
            item_category = 'non-food'
            product_type = None
        else:
            has_type = item['DFC:hasType'].split(':')[1]
            item_type = None
            product_type = None
            for prod_type in product_types_list:
                item_type = \
                    _getshare_type_from_dfc_id(has_type, dfc_ids[prod_type])
                if item_type:
                    product_type = prod_type
                    break
            item_category = 'food'
        if not item_type:
            continue

        all_text = \
            item['DFC:description'] + ' ' + item_type + ' ' + item_category
        if is_filtered_globally(base_dir, all_text):
            continue

        dfc_id = None
        if product_type:
            dfc_id = dfc_ids[product_type][item_type]
        item_id = item['@id']
        description = item['DFC:description'].split(':', 1)[1].strip()

        image_url = ''
        if item.get('DFC:Image'):
            image_url = item['DFC:Image']
        shares_json[item_id] = {
            "displayName": item['DFC:description'].split(':')[0],
            "summary": description,
            "imageUrl": image_url,
            "itemQty": float(item['DFC:quantity']),
            "dfcId": dfc_id,
            "itemType": item_type,
            "category": item_category,
            "location": "",
            "published": start_time_sec,
            "expire": expiry_time_sec,
            "itemPrice": item['DFC:price'].split(' ')[0],
            "itemCurrency": item['DFC:price'].split(' ')[1]
        }
    return shares_json


def share_category_icon(category: str) -> str:
    """Returns unicode icon for the given category
    """
    category_icons = {
        'accommodation': '🏠',
        'clothes':  '👚',
        'tools': '🔧',
        'food': '🍏'
    }
    if category_icons.get(category):
        return category_icons[category]
    return ''