__filename__ = "posts.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.5.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "ActivityPub"
import json
import html
import datetime
import os
import shutil
import sys
import time
import random
from time import gmtime, strftime
from collections import OrderedDict
from threads import thread_with_trace
from threads import begin_thread
from cache import get_actor_public_key_from_id
from cache import store_person_in_cache
from cache import get_person_from_cache
from cache import expire_person_cache
from pprint import pprint
from session import create_session
from session import get_json
from session import post_json
from session import post_json_string
from session import post_image
from session import get_json_valid
from webfinger import webfinger_handle
from httpsig import create_signed_header
from siteactive import site_is_active
from languages import understood_post_language
from utils import string_contains
from utils import get_post_attachments
from utils import is_premium_account
from utils import contains_private_key
from utils import get_url_from_post
from utils import date_from_string_format
from utils import date_epoch
from utils import date_utcnow
from utils import get_attributed_to
from utils import contains_statuses
from utils import contains_invalid_actor_url_chars
from utils import acct_handle_dir
from utils import is_dm
from utils import remove_eol
from utils import text_in_file
from utils import get_media_descriptions_from_post
from utils import valid_hash_tag
from utils import get_audio_extensions
from utils import get_summary_from_post
from utils import get_user_paths
from utils import invalid_ciphertext
from utils import has_object_string_type
from utils import remove_id_ending
from utils import replace_users_with_at
from utils import has_group_type
from utils import get_base_content_from_post
from utils import remove_domain_port
from utils import get_port_from_domain
from utils import has_object_dict
from utils import reject_post_id
from utils import remove_invalid_chars
from utils import file_last_modified
from utils import is_public_post
from utils import has_users_path
from utils import valid_post_date
from utils import get_full_domain
from utils import get_followers_list
from utils import is_evil
from utils import get_status_number
from utils import create_person_dir
from utils import url_permitted
from utils import get_nickname_from_actor
from utils import get_domain_from_actor
from utils import delete_post
from utils import valid_nickname
from utils import locate_post
from utils import load_json
from utils import save_json
from utils import get_config_param
from utils import locate_news_votes
from utils import locate_news_arrival
from utils import votes_on_newswire_item
from utils import remove_html
from utils import dangerous_markup
from utils import acct_dir
from utils import local_actor_url
from utils import get_reply_to
from utils import get_actor_from_post
from utils import data_dir
from media import get_music_metadata
from media import attach_media
from media import replace_you_tube
from media import replace_twitter
from content import reject_twitter_summary
from content import words_similarity
from content import limit_repeated_words
from content import post_tag_exists
from content import remove_long_words
from content import add_html_tags
from content import replace_emoji_from_tags
from content import remove_text_formatting
from content import add_auto_cw
from auth import create_basic_auth_header
from blocking import is_blocked_hashtag
from blocking import is_blocked
from blocking import is_blocked_domain
from filters import is_filtered
from filters import is_question_filtered
from git import convert_post_to_patch
from linked_data_sig import generate_json_signature
from petnames import resolve_petnames
from video import convert_video_to_note
from context import get_individual_post_context
from maps import get_location_dict_from_tags
from maps import geocoords_from_map_link
from keys import get_person_key
from markdown import markdown_to_html
from followerSync import update_followers_sync_cache
from question import is_question
def convert_post_content_to_html(message_json: {}) -> None:
"""Convert post content to html
"""
obj_json = message_json
if has_object_dict(message_json):
obj_json = message_json['object']
if not obj_json.get('mediaType'):
return
if not obj_json.get('content'):
return
if obj_json['mediaType'] == 'text/markdown':
content_str = obj_json['content']
obj_json['content'] = markdown_to_html(content_str)
obj_json['mediaType'] = 'text/html'
if obj_json.get('contentMap'):
langs_dict = obj_json['contentMap']
if isinstance(langs_dict, dict):
for lang, content_str in langs_dict.items():
if not isinstance(content_str, str):
continue
obj_json['contentMap'][lang] = \
markdown_to_html(content_str)
def is_moderator(base_dir: str, nickname: str) -> bool:
"""Returns true if the given nickname is a moderator
"""
moderators_file = data_dir(base_dir) + '/moderators.txt'
if not os.path.isfile(moderators_file):
admin_name = get_config_param(base_dir, 'admin')
if not admin_name:
return False
if admin_name == nickname:
return True
return False
with open(moderators_file, 'r', encoding='utf-8') as fp_mod:
lines = fp_mod.readlines()
if len(lines) == 0:
admin_name = get_config_param(base_dir, 'admin')
if not admin_name:
return False
if admin_name == nickname:
return True
for moderator in lines:
moderator = moderator.strip('\n').strip('\r')
if moderator == nickname:
return True
return False
def no_of_followers_on_domain(base_dir: str, handle: str,
domain: str, follow_file='followers.txt') -> int:
"""Returns the number of followers of the given handle from the
given domain
"""
filename = acct_handle_dir(base_dir, handle) + '/' + follow_file
if not os.path.isfile(filename):
return 0
ctr = 0
with open(filename, 'r', encoding='utf-8') as followers_file:
for follower_handle in followers_file:
if '@' in follower_handle:
follower_domain = follower_handle.split('@')[1]
follower_domain = remove_eol(follower_domain)
if domain == follower_domain:
ctr += 1
return ctr
def _clean_html(raw_html: str) -> str:
# text=BeautifulSoup(raw_html, 'html.parser').get_text()
text = raw_html
return html.unescape(text)
def get_user_url(wf_request: {}, source_id: int, debug: bool) -> str:
"""Gets the actor url from a webfinger request
"""
if not wf_request.get('links'):
if source_id == 72367:
print('get_user_url ' + str(source_id) +
' failed to get display name for webfinger ' +
str(wf_request))
else:
print('get_user_url webfinger activity+json contains no links ' +
str(source_id) + ' ' + str(wf_request))
return None
for link in wf_request['links']:
if not (link.get('type') and link.get('href')):
continue
if 'application/activity+json' not in link['type'] and \
'application/ld+json' not in link['type']:
continue
if '/@' not in link['href']:
if debug and not has_users_path(link['href']):
print('get_user_url webfinger activity+json ' +
'contains single user instance actor ' +
str(source_id) + ' ' + str(link))
url = link['href']
if not contains_invalid_actor_url_chars(url):
return remove_html(url)
return None
def parse_user_feed(signing_priv_key_pem: str,
session, feed_url: str, as_header: {},
project_version: str, http_prefix: str,
origin_domain: str, debug: bool, depth: int) -> []:
"""Returns a feed of posts for an account url
"""
if depth > 10:
if debug:
print('Maximum search depth reached')
return None
if debug:
print('Getting user feed for ' + feed_url)
print('User feed header ' + str(as_header))
print('http_prefix ' + str(http_prefix))
print('origin_domain ' + str(origin_domain))
feed_json = \
get_json(signing_priv_key_pem, session, feed_url, as_header, None,
debug, project_version, http_prefix, origin_domain)
if not get_json_valid(feed_json):
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = 'application/ld+json; profile="' + profile_str + '"'
if as_header['Accept'] != accept_str:
as_header = {
'Accept': accept_str
}
feed_json = get_json(signing_priv_key_pem, session, feed_url,
as_header, None, debug, project_version,
http_prefix, origin_domain)
if not get_json_valid(feed_json):
if debug:
print('No user feed was returned')
return None
if debug:
print('User feed:')
pprint(feed_json)
if 'orderedItems' in feed_json:
return feed_json['orderedItems']
if 'items' in feed_json:
return feed_json['items']
next_url = None
if 'first' in feed_json:
next_url = feed_json['first']
elif 'next' in feed_json:
next_url = feed_json['next']
if debug:
print('User feed next url: ' + str(next_url))
if next_url:
if isinstance(next_url, str):
if '?max_id=0' not in next_url:
user_feed = \
parse_user_feed(signing_priv_key_pem,
session, next_url, as_header,
project_version, http_prefix,
origin_domain, debug, depth + 1)
if user_feed:
return user_feed
elif isinstance(next_url, dict):
user_feed = next_url
if user_feed.get('orderedItems'):
return user_feed['orderedItems']
if user_feed.get('items'):
return user_feed['items']
return None
def _get_person_box_actor(session, base_dir: str, actor: str,
profile_str: str, as_header: {},
debug: bool, project_version: str,
http_prefix: str, origin_domain: str,
person_cache: {},
signing_priv_key_pem: str,
source_id: int) -> {}:
"""Returns the actor json for the given actor url
"""
person_json = \
get_person_from_cache(base_dir, actor, person_cache)
if person_json:
return person_json
if '/channel/' in actor or '/accounts/' in actor:
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
person_json = \
get_json(signing_priv_key_pem, session, actor, as_header, None,
debug, project_version, http_prefix, origin_domain)
if get_json_valid(person_json):
return person_json
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
person_json = \
get_json(signing_priv_key_pem, session, actor, as_header, None,
debug, project_version, http_prefix, origin_domain)
if get_json_valid(person_json):
return person_json
print('Unable to get actor for ' + actor + ' ' + str(source_id))
if not signing_priv_key_pem:
print('No signing key provided when getting actor')
return None
def get_person_box(signing_priv_key_pem: str, origin_domain: str,
base_dir: str, session, wf_request: {}, person_cache: {},
project_version: str, http_prefix: str,
nickname: str, domain: str,
box_name: str, source_id: int,
system_language: str) -> (str, str, str, str, str,
str, str, bool):
debug = False
profile_str = 'https://www.w3.org/ns/activitystreams'
as_header = {
'Accept': 'application/activity+json; profile="' + profile_str + '"'
}
if not wf_request:
print('No webfinger given')
return None, None, None, None, None, None, None, None
# get the actor / person_url
if not wf_request.get('errors'):
# get the actor url from webfinger links
person_url = get_user_url(wf_request, source_id, debug)
else:
if nickname == 'dev':
# try single user instance
print('get_person_box: Trying single user instance with ld+json')
person_url = http_prefix + '://' + domain
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
else:
# the final fallback is a mastodon style url
person_url = local_actor_url(http_prefix, nickname, domain)
if not person_url:
return None, None, None, None, None, None, None, None
# get the actor json from the url
person_json = \
_get_person_box_actor(session, base_dir, person_url,
profile_str, as_header,
debug, project_version,
http_prefix, origin_domain,
person_cache, signing_priv_key_pem,
source_id)
if not person_json:
return None, None, None, None, None, None, None, None
is_group = False
if person_json.get('type'):
if person_json['type'] == 'Group':
is_group = True
# get the url for the box/collection
box_json = None
if not person_json.get(box_name):
if person_json.get('endpoints'):
if person_json['endpoints'].get(box_name):
box_json = person_json['endpoints'][box_name]
else:
box_json = person_json[box_name]
if not box_json:
return None, None, None, None, None, None, None, None
person_id = None
if person_json.get('id'):
person_id = person_json['id']
pub_key, pub_key_id = get_actor_public_key_from_id(person_json, None)
shared_inbox = None
if person_json.get('sharedInbox'):
shared_inbox = person_json['sharedInbox']
else:
if person_json.get('endpoints'):
if person_json['endpoints'].get('sharedInbox'):
shared_inbox = person_json['endpoints']['sharedInbox']
avatar_url = None
if person_json.get('icon'):
icon_dict = person_json['icon']
if isinstance(icon_dict, list):
if len(icon_dict) > 0:
icon_dict = icon_dict[-1]
if isinstance(icon_dict, dict):
if icon_dict.get('url'):
url_str = get_url_from_post(icon_dict['url'])
avatar_url = remove_html(url_str)
display_name = None
possible_display_name = None
if person_json.get('name'):
possible_display_name = person_json['name']
if person_json.get('nameMap'):
if isinstance(person_json['nameMap'], dict):
if system_language in person_json['nameMap']:
if isinstance(person_json['nameMap'][system_language], str):
possible_display_name = \
person_json['nameMap'][system_language]
if possible_display_name:
display_name = possible_display_name
if dangerous_markup(possible_display_name, False, []):
display_name = '*ADVERSARY*'
elif is_filtered(base_dir,
nickname, domain,
possible_display_name, 'en'):
display_name = '*FILTERED*'
# have they moved?
if person_json.get('movedTo'):
display_name += ' ⌂'
store_person_in_cache(base_dir, person_url, person_json,
person_cache, True)
return box_json, pub_key_id, pub_key, person_id, shared_inbox, \
avatar_url, display_name, is_group
def _is_public_feed_post(item: {}, person_posts: {}, debug: bool) -> bool:
"""Is the given post a public feed post?
"""
if not isinstance(item, dict):
if debug:
print('item object is not a dict')
pprint(item)
return False
if not item.get('id'):
if debug:
print('No id')
return False
if not item.get('type'):
if debug:
print('No type')
return False
allowed_post_types = ('Create', 'Announce', 'Page', 'Note', 'Event')
if item['type'] not in allowed_post_types:
if debug:
print('Not a Create/Note/Announce type')
return False
if item.get('object'):
if isinstance(item['object'], dict):
if not item['object'].get('published'):
if debug:
print('No published attribute 1')
return False
elif isinstance(item['object'], str):
if item['type'] != 'Announce' and \
not item.get('published'):
if debug:
print('No published attribute 2')
return False
else:
if debug:
print('object is not a dict or string')
return False
elif item['type'] in ('Note', 'Event', 'Page'):
if not item.get('published'):
if debug:
print('No published attribute 3')
return False
if not person_posts.get(item['id']):
this_item = item
if item.get('object'):
this_item = item['object']
# check that this is a public post
# #Public should appear in the "to" list
item_is_note = False
if item['type'] in ('Note', 'Event', 'Page'):
item_is_note = True
if isinstance(this_item, dict):
if this_item.get('to'):
is_public = False
if isinstance(this_item['to'], list):
for recipient in this_item['to']:
if recipient.endswith('#Public') or \
recipient == 'as:Public' or \
recipient == 'Public':
is_public = True
break
elif isinstance(this_item['to'], str):
recipient = this_item['to']
if recipient.endswith('#Public') or \
recipient == 'as:Public' or \
recipient == 'Public':
is_public = True
if not is_public:
return False
elif isinstance(this_item, str) or item_is_note:
if item.get('to'):
is_public = False
if isinstance(item['to'], list):
for recipient in item['to']:
if recipient.endswith('#Public') or \
recipient == 'as:Public' or \
recipient == 'Public':
is_public = True
break
elif isinstance(item['to'], str):
recipient = item['to']
if recipient.endswith('#Public') or \
recipient == 'as:Public' or \
recipient == 'Public':
is_public = True
if not is_public:
return False
return True
def is_create_inside_announce(item: {}) -> bool:
""" is this a Create inside of an Announce?
eg. lemmy feed item
"""
if not isinstance(item, dict):
return False
if not item.get('type'):
return False
if item['type'] != 'Announce':
return False
if not item.get('object'):
return False
if not isinstance(item['object'], dict):
return False
if not item['object'].get('type'):
return False
if item['object']['type'] != 'Create':
return False
return True
def _get_posts(session, outbox_url: str, max_posts: int,
max_mentions: int,
max_emoji: int, max_attachments: int,
federation_list: [], raw: bool,
simple: bool, debug: bool,
project_version: str, http_prefix: str,
origin_domain: str, system_language: str,
signing_priv_key_pem: str) -> {}:
"""Gets public posts from an outbox
"""
if debug:
print('Getting outbox posts for ' + outbox_url)
person_posts = {}
if not outbox_url:
return person_posts
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if raw:
if debug:
print('Returning the raw feed')
result = []
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix,
origin_domain, debug, 0)
if user_feed:
for item in user_feed:
result.append(item)
i += 1
if i == max_posts:
break
pprint(result)
return None
if debug:
print('Returning a human readable version of the feed')
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix,
origin_domain, debug, 0)
if not user_feed:
return person_posts
i = 0
for item in user_feed:
if is_create_inside_announce(item):
item = item['object']
if not _is_public_feed_post(item, person_posts, debug):
continue
this_item = item
this_item_type = item['type']
if this_item_type not in ('Note', 'Event', 'Page'):
this_item = item['object']
if isinstance(this_item, str):
if '://' in this_item:
profile_str = 'https://www.w3.org/ns/activitystreams'
as_header2_str = 'application/ld+json; profile="' + \
profile_str + '"'
as_header2 = {
'Accept': as_header2_str
}
this_item = \
get_json(signing_priv_key_pem, session, this_item,
as_header2, None, debug, __version__,
http_prefix, origin_domain)
if not get_json_valid(this_item):
continue
content = get_base_content_from_post(this_item, system_language)
if not content:
continue
content = content.replace(''', "'")
mentions = []
emoji = {}
summary = ''
in_reply_to = ''
attachment = []
sensitive = False
if isinstance(this_item, dict):
if this_item.get('tag'):
for tag_item in this_item['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'emoji':
if tag_item.get('name') and tag_item.get('icon'):
if tag_item['icon'].get('url'):
# No emoji from non-permitted domains
url_str = \
get_url_from_post(tag_item['icon']['url'])
if url_permitted(url_str,
federation_list):
emoji_name = tag_item['name']
emoji_icon = remove_html(url_str)
emoji[emoji_name] = emoji_icon
else:
if debug:
print('url not permitted ' + url_str)
if tag_type == 'mention':
if tag_item.get('name'):
if tag_item['name'] not in mentions:
mentions.append(tag_item['name'])
if len(mentions) > max_mentions:
if debug:
print('max mentions reached')
continue
if len(emoji) > max_emoji:
if debug:
print('max emojis reached')
continue
if this_item.get('summaryMap'):
if this_item['summaryMap'].get(system_language):
summary = this_item['summaryMap'][system_language]
if not summary and this_item.get('summary'):
if this_item['summary']:
summary = this_item['summary']
reply_id = get_reply_to(this_item)
if reply_id:
if isinstance(reply_id, str):
# No replies to non-permitted domains
if not url_permitted(reply_id,
federation_list):
if debug:
print('url not permitted ' +
reply_id)
continue
in_reply_to = reply_id
post_attachments = get_post_attachments(this_item)
if post_attachments:
if len(post_attachments) > max_attachments:
if debug:
print('max attachments reached')
continue
if post_attachments:
for attach in post_attachments:
if attach.get('name') and attach.get('url'):
# no attachments from non-permitted domains
url_str = get_url_from_post(attach['url'])
attach_url = remove_html(url_str)
if url_permitted(attach_url,
federation_list):
attachment.append([attach['name'],
attach_url])
else:
if debug:
print('url not permitted ' + url_str)
sensitive = False
if this_item.get('sensitive'):
sensitive = this_item['sensitive']
if content:
if simple:
print(_clean_html(content) + '\n')
else:
pprint(item)
person_posts[this_item['id']] = {
"sensitive": sensitive,
"inreplyto": in_reply_to,
"summary": summary,
"html": content,
"plaintext": _clean_html(content),
"attachment": attachment,
"mentions": mentions,
"emoji": emoji
}
i += 1
if i == max_posts:
break
return person_posts
def _get_common_words() -> str:
"""Returns a list of common words
"""
return (
'that', 'some', 'about', 'then', 'they', 'were',
'also', 'from', 'with', 'this', 'have', 'more',
'need', 'here', 'would', 'these', 'into', 'very',
'well', 'when', 'what', 'your', 'there', 'which',
'even', 'there', 'such', 'just', 'those', 'only',
'will', 'much', 'than', 'them', 'each', 'goes',
'been', 'over', 'their', 'where', 'could', 'though',
'like', 'think', 'same', 'maybe', 'really', 'thing',
'something', 'possible', 'actual', 'actually',
'because', 'around', 'having', 'especially', 'other',
'making', 'made', 'make', 'makes', 'including',
'includes', 'know', 'knowing', 'knows', 'things',
'say', 'says', 'saying', 'many', 'somewhat',
'problem', 'problems', 'idea', 'ideas',
'using', 'uses', 'https', 'still', 'want', 'wants'
)
def _update_word_frequency(content: str, word_frequency: {}) -> None:
"""Creates a dictionary containing words and the number of times
that they appear
"""
plain_text = remove_html(content)
remove_chars = ('.', ';', '?', '\n', ':')
for char in remove_chars:
plain_text = plain_text.replace(char, ' ')
words_list = plain_text.split(' ')
common_words = _get_common_words()
for word in words_list:
word_len = len(word)
if word_len < 3:
continue
if word_len < 4:
if word.upper() != word:
continue
if string_contains(word, ('&', '"', '@', "'", "--", '//')):
continue
if word.lower() in common_words:
continue
if word_frequency.get(word):
word_frequency[word] += 1
else:
word_frequency[word] = 1
def get_post_domains(session, outbox_url: str, max_posts: int, debug: bool,
project_version: str, http_prefix: str, domain: str,
word_frequency: {}, domain_list: [],
system_language: str, signing_priv_key_pem: str) -> []:
"""Returns a list of domains referenced within public posts
"""
if not outbox_url:
return []
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
post_domains = domain_list
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix, domain,
debug, 0)
if not user_feed:
return post_domains
for item in user_feed:
i += 1
if i > max_posts:
break
if not has_object_dict(item):
continue
content_str = get_base_content_from_post(item, system_language)
if content_str:
_update_word_frequency(content_str, word_frequency)
reply_id = get_reply_to(item['object'])
if reply_id:
if isinstance(reply_id, str):
post_domain, _ = \
get_domain_from_actor(reply_id)
if post_domain:
if post_domain not in post_domains:
post_domains.append(post_domain)
if item['object'].get('tag'):
for tag_item in item['object']['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'mention':
if tag_item.get('href'):
tag_url = remove_html(tag_item['href'])
post_domain, _ = \
get_domain_from_actor(tag_url)
if post_domain:
if post_domain not in post_domains:
post_domains.append(post_domain)
return post_domains
def _get_posts_for_blocked_domains(base_dir: str,
session, outbox_url: str, max_posts: int,
debug: bool,
project_version: str, http_prefix: str,
domain: str,
signing_priv_key_pem: str) -> {}:
"""Returns a dictionary of posts for blocked domains
"""
if not outbox_url:
return {}
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
blocked_posts = {}
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix, domain,
debug, 0)
if not user_feed:
return blocked_posts
for item in user_feed:
i += 1
if i > max_posts:
break
if not has_object_dict(item):
continue
reply_id = get_reply_to(item['object'])
if reply_id:
if isinstance(reply_id, str):
post_domain, _ = \
get_domain_from_actor(reply_id)
if not post_domain:
continue
if is_blocked_domain(base_dir, post_domain, None, None):
if item['object'].get('url'):
url = get_url_from_post(item['object']['url'])
else:
url = get_url_from_post(item['object']['id'])
url = remove_html(url)
if not blocked_posts.get(post_domain):
blocked_posts[post_domain] = [url]
else:
if url not in blocked_posts[post_domain]:
blocked_posts[post_domain].append(url)
if item['object'].get('tag'):
for tag_item in item['object']['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'mention' and tag_item.get('href'):
tag_url = remove_html(tag_item['href'])
post_domain, _ = \
get_domain_from_actor(tag_url)
if not post_domain:
continue
if is_blocked_domain(base_dir, post_domain, None, None):
if item['object'].get('url'):
url = get_url_from_post(item['object']['url'])
else:
url = get_url_from_post(item['object']['id'])
url = remove_html(url)
if not blocked_posts.get(post_domain):
blocked_posts[post_domain] = [url]
else:
if url not in blocked_posts[post_domain]:
blocked_posts[post_domain].append(url)
return blocked_posts
def delete_all_posts(base_dir: str,
nickname: str, domain: str, boxname: str) -> None:
"""Deletes all posts for a person from inbox or outbox
"""
if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews'):
return
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
for delete_filename in os.scandir(box_dir):
delete_filename = delete_filename.name
file_path = os.path.join(box_dir, delete_filename)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path, ignore_errors=False, onerror=None)
except OSError as ex:
print('ERROR: delete_all_posts ' + str(ex))
def _save_last_published(base_dir: str, nickname: str, domain: str,
published: str) -> None:
"""Saves the last published date for the outbox
"""
if not published:
return
published_filename = \
acct_dir(base_dir, nickname, domain) + '/.last_published'
try:
with open(published_filename, 'w+', encoding='utf-8') as fp_last:
fp_last.write(published)
except OSError:
print('EX: unable to save last published time ' +
published_filename)
def save_post_to_box(base_dir: str, http_prefix: str, post_id: str,
nickname: str, domain: str, post_json_object: {},
boxname: str) -> str:
"""Saves the give json to the give box
Returns the filename
"""
if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews', 'scheduled'):
return None
original_domain = domain
domain = remove_domain_port(domain)
if not post_id:
status_number, _ = get_status_number()
post_id = \
local_actor_url(http_prefix, nickname, original_domain) + \
'/statuses/' + status_number
post_json_object['id'] = post_id + '/activity'
published = None
if post_json_object.get('published'):
published = post_json_object['published']
if has_object_dict(post_json_object):
post_json_object['object']['id'] = post_id
post_json_object['object']['atomUri'] = post_id
if post_json_object['object'].get('published'):
published = post_json_object['object']['published']
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
filename = box_dir + '/' + post_id.replace('/', '#') + '.json'
save_json(post_json_object, filename)
# if this is an outbox post with a duplicate in the inbox then save to both
# This happens for edited posts
if '/outbox/' in filename:
_save_last_published(base_dir, nickname, domain, published)
inbox_filename = filename.replace('/outbox/', '/inbox/')
if os.path.isfile(inbox_filename):
save_json(post_json_object, inbox_filename)
base_filename = \
filename.replace('/outbox/',
'/postcache/').replace('.json', '')
ssml_filename = base_filename + '.ssml'
if os.path.isfile(ssml_filename):
try:
os.remove(ssml_filename)
except OSError:
print('EX: save_post_to_box unable to delete ssml file ' +
ssml_filename)
html_filename = base_filename + '.html'
if os.path.isfile(html_filename):
try:
os.remove(html_filename)
except OSError:
print('EX: save_post_to_box unable to delete html file ' +
html_filename)
return filename
def _update_hashtags_index(base_dir: str, tag: {}, new_post_id: str,
nickname: str) -> None:
"""Writes the post url for hashtags to a file
This allows posts for a hashtag to be quickly looked up
"""
if tag['type'] != 'Hashtag':
return
# create hashtags directory
tags_dir = base_dir + '/tags'
if not os.path.isdir(tags_dir):
os.mkdir(tags_dir)
tag_name = tag['name']
tags_filename = tags_dir + '/' + tag_name[1:] + '.txt'
new_post_id = new_post_id.replace('/', '#')
if not os.path.isfile(tags_filename):
days_diff = date_utcnow() - date_epoch()
days_since_epoch = days_diff.days
tag_line = \
str(days_since_epoch) + ' ' + nickname + ' ' + \
new_post_id + '\n'
# create a new tags index file
try:
with open(tags_filename, 'w+', encoding='utf-8') as tags_file:
tags_file.write(tag_line)
except OSError:
print('EX: _update_hashtags_index unable to write tags file ' +
tags_filename)
else:
# prepend to tags index file
if not text_in_file(new_post_id, tags_filename):
days_diff = date_utcnow() - date_epoch()
days_since_epoch = days_diff.days
tag_line = \
str(days_since_epoch) + ' ' + nickname + ' ' + \
new_post_id + '\n'
try:
with open(tags_filename, 'r+', encoding='utf-8') as tags_file:
content = tags_file.read()
if tag_line not in content:
tags_file.seek(0, 0)
tags_file.write(tag_line + content)
except OSError as ex:
print('EX: Failed to write entry to tags file ' +
tags_filename + ' ' + str(ex))
def _add_schedule_post(base_dir: str, nickname: str, domain: str,
event_date_str: str, post_id: str) -> None:
"""Adds a scheduled post to the index
"""
handle = nickname + '@' + domain
schedule_index_filename = \
acct_handle_dir(base_dir, handle) + '/schedule.index'
index_str = event_date_str + ' ' + post_id.replace('/', '#')
if os.path.isfile(schedule_index_filename):
if not text_in_file(index_str, schedule_index_filename):
try:
with open(schedule_index_filename, 'r+',
encoding='utf-8') as schedule_file:
content = schedule_file.read()
if index_str + '\n' not in content:
schedule_file.seek(0, 0)
schedule_file.write(index_str + '\n' + content)
print('DEBUG: scheduled post added to index')
except OSError as ex:
print('EX: Failed to write entry to scheduled posts index ' +
schedule_index_filename + ' ' + str(ex))
else:
try:
with open(schedule_index_filename, 'w+',
encoding='utf-8') as schedule_file:
schedule_file.write(index_str + '\n')
except OSError as ex:
print('EX: Failed to write entry to scheduled posts index2 ' +
schedule_index_filename + ' ' + str(ex))
def valid_content_warning(summary: str) -> str:
"""Returns a validated content warning
"""
cw_str = remove_html(summary)
# hashtags within content warnings apparently cause a lot of trouble
# so remove them
if '#' in cw_str:
cw_str = cw_str.replace('#', '').replace(' ', ' ')
return remove_invalid_chars(cw_str)
def _create_post_cw_from_reply(base_dir: str, nickname: str, domain: str,
in_reply_to: str,
sensitive: bool, summary: str,
system_language: str,
languages_understood: []) -> (bool, str):
"""If this is a reply and the original post has a CW
then use the same CW
"""
reply_to_json = None
if in_reply_to and not sensitive:
# locate the post which this is a reply to and check if
# it has a content warning. If it does then reproduce
# the same warning
reply_post_filename = \
locate_post(base_dir, nickname, domain, in_reply_to)
if reply_post_filename:
reply_to_json = load_json(reply_post_filename)
if reply_to_json:
if reply_to_json.get('object'):
if reply_to_json['object'].get('sensitive'):
if reply_to_json['object']['sensitive']:
sensitive = True
if reply_to_json['object'].get('summary'):
summary = \
get_summary_from_post(reply_to_json,
system_language,
languages_understood)
return sensitive, summary
def _attach_post_license(post_json_object: {},
content_license_url: str) -> None:
"""Attaches a license to each post
Also see:
https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-c118.md
NOTE: at present (Jan 2023) there is no consensus about how to
represent license information on ActivityPub posts, so this might
need to change if such a consensus appears.
"""
if not content_license_url:
return
post_json_object['attachment'].append({
"type": "PropertyValue",
"name": "license",
"value": content_license_url
})
post_json_object['schema:license'] = content_license_url
post_json_object['license'] = content_license_url
def _attach_buy_link(post_json_object: {},
buy_url: str, translate: {}) -> None:
"""Attaches a link for buying something
"""
if not buy_url:
return
if '://' not in buy_url:
return
if ' ' in buy_url or '<' in buy_url:
return
buy_str = 'Buy'
if translate.get(buy_str):
buy_str = translate[buy_str]
if 'attachment' not in post_json_object:
post_json_object['attachment'] = []
post_json_object['attachment'].append({
"type": "Link",
"name": buy_str,
"href": buy_url,
"rel": "payment",
"mediaType": "text/html"
})
def _attach_chat_link(post_json_object: {},
chat_url: str) -> None:
"""Attaches a chat link
"""
if not chat_url:
return
if '://' not in chat_url:
return
if ' ' in chat_url or '<' in chat_url:
return
if 'attachment' not in post_json_object:
post_json_object['attachment'] = []
post_json_object['attachment'].append({
"type": "Link",
"name": "Chat",
"href": chat_url,
"rel": "discussion"
})
def _create_post_s2s(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, content: str, status_number: str,
published: str, new_post_id: str, post_context: {},
to_recipients: [], to_cc: [], in_reply_to: str,
sensitive: bool, comments_enabled: bool,
tags: [], attach_image_filename: str,
media_type: str, image_description: str,
video_transcript: str, city: str,
post_object_type: str, summary: str,
in_reply_to_atom_uri: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
buy_url: str, chat_url: str, translate: {}) -> {}:
"""Creates a new server-to-server post
"""
actor_url = local_actor_url(http_prefix, nickname, domain)
id_str = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number + '/replies'
new_post_url = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
new_post_attributed_to = \
local_actor_url(http_prefix, nickname, domain)
if not conversation_id:
conversation_id = new_post_id
new_post = {
'@context': post_context,
'id': new_post_id + '/activity',
'type': 'Create',
'actor': actor_url,
'published': published,
'to': to_recipients,
'cc': to_cc,
'object': {
'id': new_post_id,
'conversation': conversation_id,
'context': conversation_id,
'type': post_object_type,
'summary': summary,
'inReplyTo': in_reply_to,
'published': published,
'url': new_post_url,
'attributedTo': new_post_attributed_to,
'to': to_recipients,
'cc': to_cc,
'sensitive': sensitive,
'atomUri': new_post_id,
'inReplyToAtomUri': in_reply_to_atom_uri,
'commentsEnabled': comments_enabled,
'rejectReplies': not comments_enabled,
'mediaType': 'text/html',
'content': content,
'contentMap': {
system_language: content
},
'attachment': [],
'tag': tags,
'replies': {
'id': id_str,
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'next': id_str + '?only_other_accounts=true&page=true',
'partOf': id_str,
'items': []
}
},
"crawlable": False
}
}
# pixelfed/friendica style location representation
location = get_location_dict_from_tags(tags)
if location:
if location.get('name') and \
location.get('longitude') and \
location.get('latitude'):
new_post['object']['location'] = {
'type': 'Place',
'name': location['name'],
'longitude': location['longitude'],
'latitude': location['latitude']
}
elif location.get('name'):
new_post['object']['location'] = {
'type': 'Place',
'name': location['name']
}
if attach_image_filename:
new_post['object'] = \
attach_media(base_dir, http_prefix, nickname, domain, port,
new_post['object'], attach_image_filename,
media_type, image_description, video_transcript,
city, low_bandwidth,
media_license_url, media_creator, system_language)
_attach_post_license(new_post['object'], content_license_url)
_attach_buy_link(new_post['object'], buy_url, translate)
_attach_chat_link(new_post['object'], chat_url)
return new_post
def _create_post_c2s(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, content: str, status_number: str,
published: str, new_post_id: str, post_context: {},
to_recipients: [], to_cc: [], in_reply_to: str,
sensitive: bool, comments_enabled: bool,
tags: [], attach_image_filename: str,
media_type: str, image_description: str,
video_transcript: str, city: str,
post_object_type: str, summary: str,
in_reply_to_atom_uri: str, system_language: str,
conversation_id: str, low_bandwidth: str,
content_license_url: str, media_license_url: str,
media_creator: str, buy_url: str, chat_url: str,
translate: {}) -> {}:
"""Creates a new client-to-server post
"""
domain_full = get_full_domain(domain, port)
id_str = \
local_actor_url(http_prefix, nickname, domain_full) + \
'/statuses/' + status_number + '/replies'
new_post_url = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
if not conversation_id:
conversation_id = new_post_id
new_post = {
"@context": post_context,
'id': new_post_id,
'conversation': conversation_id,
'context': conversation_id,
'type': post_object_type,
'summary': summary,
'inReplyTo': in_reply_to,
'published': published,
'url': new_post_url,
'attributedTo': local_actor_url(http_prefix, nickname, domain_full),
'to': to_recipients,
'cc': to_cc,
'sensitive': sensitive,
'atomUri': new_post_id,
'inReplyToAtomUri': in_reply_to_atom_uri,
'commentsEnabled': comments_enabled,
'rejectReplies': not comments_enabled,
'mediaType': 'text/html',
'content': content,
'contentMap': {
system_language: content
},
'attachment': [],
'tag': tags,
'replies': {
'id': id_str,
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'next': id_str + '?only_other_accounts=true&page=true',
'partOf': id_str,
'items': []
}
},
"crawlable": False
}
# pixelfed/friendica style location representation
location = get_location_dict_from_tags(tags)
if location:
if location.get('name') and \
location.get('longitude') and \
location.get('latitude'):
new_post['location'] = {
'type': 'Place',
'name': location['name'],
'longitude': location['longitude'],
'latitude': location['latitude']
}
elif location.get('name'):
new_post['location'] = {
'type': 'Place',
'name': location['name']
}
if attach_image_filename:
new_post = \
attach_media(base_dir, http_prefix, nickname, domain, port,
new_post, attach_image_filename,
media_type, image_description, video_transcript,
city, low_bandwidth,
media_license_url, media_creator,
system_language)
_attach_post_license(new_post, content_license_url)
_attach_buy_link(new_post, buy_url, translate)
_attach_chat_link(new_post, chat_url)
return new_post
def _create_post_place_and_time(event_date: str, end_date: str,
event_time: str, end_time: str,
summary: str, content: str,
schedule_post: bool,
event_uuid: str,
location: str,
tags: []) -> str:
"""Adds a place and time to the tags on a new post
"""
end_date_str = None
if end_date:
event_name = summary
if not event_name:
event_name = content
end_date_str = end_date
if end_time:
if end_time.endswith('Z'):
end_date_str = end_date + 'T' + end_time
else:
end_date_str = end_date + 'T' + end_time + \
':00' + strftime("%z", gmtime())
else:
if not event_time:
end_date_str = end_date + 'T12:00:00Z'
else:
if event_time.endswith('Z'):
end_date_str = end_date + 'T' + event_time
else:
end_date_str = \
end_date + 'T' + event_time + ':00' + \
strftime("%z", gmtime())
# get the starting date and time
event_date_str = None
if event_date:
event_name = summary
if not event_name:
event_name = content
event_date_str = event_date
if event_time:
if event_time.endswith('Z'):
event_date_str = event_date + 'T' + event_time
else:
event_date_str = event_date + 'T' + event_time + \
':00' + strftime("%z", gmtime())
else:
event_date_str = event_date + 'T12:00:00Z'
if not end_date_str:
end_date_str = event_date_str
if not schedule_post and not event_uuid:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Event",
"name": event_name,
"startTime": event_date_str,
"endTime": end_date_str
})
if location and not event_uuid:
latitude = longitude = None
if '://' in location:
_, latitude, longitude = \
geocoords_from_map_link(location, 'openstreetmap.org')
if latitude and longitude:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Place",
"name": location,
"latitude": latitude,
"longitude": longitude
})
else:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Place",
"name": location
})
return event_date_str
def _consolidate_actors_list(actors_list: []) -> None:
""" consolidate duplicated actors
https://domain/@nick gets merged with https://domain/users/nick
"""
possible_duplicate_actors = []
for cc_actor in actors_list:
if '/@' in cc_actor:
if '/@/' not in cc_actor:
if cc_actor not in possible_duplicate_actors:
possible_duplicate_actors.append(cc_actor)
if possible_duplicate_actors:
u_paths = get_user_paths()
remove_actors = []
for cc_actor in possible_duplicate_actors:
for usr_path in u_paths:
if '/@/' not in cc_actor:
cc_actor_full = cc_actor.replace('/@', usr_path)
else:
cc_actor_full = cc_actor
if cc_actor_full in actors_list:
if cc_actor not in remove_actors:
remove_actors.append(cc_actor)
break
for cc_actor in remove_actors:
actors_list.remove(cc_actor)
def _create_post_mentions(cc_url: str, new_post: {},
to_recipients: [], tags: []) -> None:
"""Updates mentions for a new post
"""
if not cc_url:
return
if len(cc_url) == 0:
return
if new_post.get('object'):
if cc_url not in new_post['object']['cc']:
new_post['object']['cc'] = [cc_url] + new_post['object']['cc']
# if this is a public post then include any mentions in cc
to_cc = new_post['object']['cc']
if len(to_recipients) != 1:
return
to_public_recipient = False
if to_recipients[0].endswith('#Public') or \
to_recipients[0] == 'as:Public' or \
to_recipients[0] == 'Public':
to_public_recipient = True
if to_public_recipient and \
cc_url.endswith('/followers'):
for tag in tags:
if tag['type'] != 'Mention':
continue
if tag['href'] not in to_cc:
tag_url = remove_html(tag['href'])
new_post['object']['cc'].append(tag_url)
_consolidate_actors_list(new_post['object']['cc'])
new_post['cc'] = new_post['object']['cc']
else:
if cc_url not in new_post['cc']:
new_post['cc'] = [cc_url] + new_post['cc']
_consolidate_actors_list(['cc'])
def _create_post_mod_report(base_dir: str,
is_moderation_report: bool, new_post: {},
new_post_id: str) -> None:
""" if this is a moderation report then add a status
"""
if not is_moderation_report:
return
# add status
if new_post.get('object'):
new_post['object']['moderationStatus'] = 'pending'
else:
new_post['moderationStatus'] = 'pending'
# save to index file
moderation_index_file = data_dir(base_dir) + '/moderation.txt'
try:
with open(moderation_index_file, 'a+', encoding='utf-8') as mod_file:
mod_file.write(new_post_id + '\n')
except OSError:
print('EX: unable to write moderation index file ' +
moderation_index_file)
def get_actor_from_in_reply_to(in_reply_to: str) -> str:
"""Tries to get the replied to actor from the inReplyTo post id
Note: this will not always be successful for some instance types
"""
reply_nickname = get_nickname_from_actor(in_reply_to)
if not reply_nickname:
return None
reply_actor = None
if '/' + reply_nickname + '/' in in_reply_to:
reply_actor = \
in_reply_to.split('/' + reply_nickname + '/')[0] + \
'/' + reply_nickname
elif '#' + reply_nickname + '#' in in_reply_to:
reply_actor = \
in_reply_to.split('#' + reply_nickname + '#')[0] + \
'#' + reply_nickname
reply_actor = reply_actor.replace('#', '/')
if not reply_actor:
return None
if '://' not in reply_actor:
return None
return reply_actor
def _create_post_base(base_dir: str,
nickname: str, domain: str, port: int,
to_url: str, cc_url: str, http_prefix: str, content: str,
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str,
media_type: str, image_description: str,
video_transcript: str, city: str,
is_moderation_report: bool,
is_article: bool,
in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str,
location: str,
event_uuid: str, category: str,
join_mode: str,
end_date: str, end_time: str,
maximum_attendee_capacity: int,
replies_moderation_option: str,
anonymous_participation_enabled: bool,
event_status: str, ticket_url: str,
system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str,
auto_cw_cache: {}) -> {}:
"""Creates a message
"""
content = remove_invalid_chars(content)
subject = add_auto_cw(base_dir, nickname, domain, subject, content,
auto_cw_cache)
if nickname != 'news':
mentioned_recipients = \
get_mentioned_people(base_dir, http_prefix, content, domain, False)
else:
mentioned_recipients = ''
# add hashtags from audio file ID3 tags, such as Artist, Album, etc
if attach_image_filename and media_type:
audio_types = get_audio_extensions()
music_metadata = None
for ext in audio_types:
if ext in media_type:
music_metadata = get_music_metadata(attach_image_filename)
break
if music_metadata:
for audio_tag, audio_value in music_metadata.items():
if audio_tag in ('title', 'track'):
continue
# capitalize and remove any spaces
audio_value = audio_value.title().replace(' ', '')
# check that the tag is valid
if valid_hash_tag(audio_value) and \
'#' + audio_value not in content:
# check that it hasn't been blocked
if not is_blocked_hashtag(base_dir, audio_value):
content += ' #' + audio_value
tags = []
hashtags_dict = {}
domain = get_full_domain(domain, port)
# add tags
if nickname != 'news':
content = \
add_html_tags(base_dir, http_prefix,
nickname, domain, content,
mentioned_recipients,
hashtags_dict, translate, True)
# replace emoji with unicode
tags = []
for tag_name, tag in hashtags_dict.items():
tags.append(tag)
# get list of tags
if nickname != 'news':
content = \
replace_emoji_from_tags(None, base_dir, content, tags, 'content',
False, True)
# remove replaced emoji
hashtags_dict_copy = hashtags_dict.copy()
for tag_name, tag in hashtags_dict_copy.items():
if tag.get('name'):
if tag['name'].startswith(':'):
if tag['name'] not in content:
del hashtags_dict[tag_name]
status_number, published = get_status_number()
new_post_id = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
sensitive = False
summary = None
if subject:
summary = remove_invalid_chars(valid_content_warning(subject))
sensitive = True
to_recipients = []
to_cc = []
if to_url:
if not isinstance(to_url, str):
print('ERROR: to_url is not a string')
return None
to_recipients = [to_url]
# who to send to
if mentioned_recipients:
for mention in mentioned_recipients:
if mention not in to_cc:
to_cc.append(mention)
is_public = False
for recipient in to_recipients:
if recipient.endswith('#Public') or \
recipient == 'as:Public' or \
recipient == 'Public':
is_public = True
break
# create a list of hashtags
# Only posts which are #Public are searchable by hashtag
if hashtags_dict:
for tag_name, tag in hashtags_dict.items():
if not post_tag_exists(tag['type'], tag['name'], tags):
tags.append(tag)
if is_public:
_update_hashtags_index(base_dir, tag, new_post_id, nickname)
# print('Content tags: ' + str(tags))
sensitive, summary = \
_create_post_cw_from_reply(base_dir, nickname, domain,
in_reply_to, sensitive, summary,
system_language, languages_understood)
event_date_str = \
_create_post_place_and_time(event_date, end_date,
event_time, end_time,
summary, content, schedule_post,
event_uuid, location, tags)
post_context = get_individual_post_context()
if not is_public:
# make sure that CC doesn't also contain a To address
# eg. To: [ "https://mydomain/users/foo/followers" ]
# CC: [ "X", "Y", "https://mydomain/users/foo", "Z" ]
remove_from_cc = []
for cc_recipient in to_cc:
for send_to_actor in to_recipients:
if cc_recipient in send_to_actor and \
cc_recipient not in remove_from_cc:
remove_from_cc.append(cc_recipient)
break
for cc_removal in remove_from_cc:
to_cc.remove(cc_removal)
else:
if in_reply_to:
# If this is a public post then get the actor being
# replied to end ensure that it is within the CC list
reply_actor = get_actor_from_in_reply_to(in_reply_to)
if reply_actor:
if reply_actor not in to_cc:
to_cc.append(reply_actor)
# the type of post to be made
post_object_type = 'Note'
if is_article:
post_object_type = 'Article'
if not client_to_server:
new_post = \
_create_post_s2s(base_dir, nickname, domain, port,
http_prefix, content, status_number,
published, new_post_id, post_context,
to_recipients, to_cc, in_reply_to,
sensitive, comments_enabled,
tags, attach_image_filename,
media_type, image_description,
video_transcript, city,
post_object_type, summary,
in_reply_to_atom_uri, system_language,
conversation_id, low_bandwidth,
content_license_url, media_license_url,
media_creator, buy_url, chat_url,
translate)
else:
new_post = \
_create_post_c2s(base_dir, nickname, domain, port,
http_prefix, content, status_number,
published, new_post_id, post_context,
to_recipients, to_cc, in_reply_to,
sensitive, comments_enabled,
tags, attach_image_filename,
media_type, image_description,
video_transcript, city,
post_object_type, summary,
in_reply_to_atom_uri, system_language,
conversation_id, low_bandwidth,
content_license_url, media_license_url,
media_creator, buy_url, chat_url,
translate)
_create_post_mentions(cc_url, new_post, to_recipients, tags)
_create_post_mod_report(base_dir, is_moderation_report,
new_post, new_post_id)
# If a patch has been posted - i.e. the output from
# git format-patch - then convert the activitypub type
convert_post_to_patch(base_dir, nickname, domain, new_post)
if schedule_post:
if event_date and event_time:
# add an item to the scheduled post index file
_add_schedule_post(base_dir, nickname, domain,
event_date_str, new_post_id)
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'scheduled')
else:
print('Unable to create scheduled post without ' +
'date and time values')
return new_post
elif save_to_file:
if is_article:
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'tlblogs')
else:
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'outbox')
return new_post
def outbox_message_create_wrap(http_prefix: str,
nickname: str, domain: str, port: int,
message_json: {}) -> {}:
"""Wraps a received message in a Create
https://www.w3.org/TR/activitypub/#object-without-create
"""
domain = get_full_domain(domain, port)
status_number, published = get_status_number()
if message_json.get('published'):
published = message_json['published']
new_post_id = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
cc_list = []
if message_json.get('cc'):
cc_list = message_json['cc']
new_post = {
"@context": "https://www.w3.org/ns/activitystreams",
'id': new_post_id + '/activity',
'type': 'Create',
'actor': local_actor_url(http_prefix, nickname, domain),
'published': published,
'to': message_json['to'],
'cc': cc_list,
'object': message_json
}
new_post['object']['id'] = new_post['id']
new_post['object']['url'] = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
new_post['object']['atomUri'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
return new_post
def _post_is_addressed_to_followers(nickname: str, domain: str, port: int,
http_prefix: str,
post_json_object: {}) -> bool:
"""Returns true if the given post is addressed to followers of the nickname
"""
domain_full = get_full_domain(domain, port)
if not post_json_object.get('object'):
return False
to_list = []
cc_list = []
if post_json_object['type'] != 'Update' and \
has_object_dict(post_json_object):
if post_json_object['object'].get('to'):
if isinstance(post_json_object['object']['to'], list):
to_list = post_json_object['object']['to']
elif isinstance(post_json_object['object']['to'], str):
to_list = [post_json_object['object']['to']]
if post_json_object['object'].get('cc'):
if isinstance(post_json_object['object']['cc'], list):
cc_list = post_json_object['object']['cc']
elif isinstance(post_json_object['object']['cc'], str):
cc_list = [post_json_object['object']['cc']]
else:
if post_json_object.get('to'):
if isinstance(post_json_object['to'], list):
to_list = post_json_object['to']
elif isinstance(post_json_object['to'], str):
to_list = [post_json_object['to']]
if post_json_object.get('cc'):
if isinstance(post_json_object['cc'], list):
cc_list = post_json_object['cc']
elif isinstance(post_json_object['cc'], str):
cc_list = [post_json_object['cc']]
followers_url = \
local_actor_url(http_prefix, nickname, domain_full) + '/followers'
# does the followers url exist in 'to' or 'cc' lists?
addressed_to_followers = False
if followers_url in to_list:
addressed_to_followers = True
elif followers_url in cc_list:
addressed_to_followers = True
return addressed_to_followers
def pin_post2(base_dir: str, nickname: str, domain: str,
pinned_content: str) -> None:
"""Pins the given post Id to the profile of then given account
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
try:
with open(pinned_filename, 'w+', encoding='utf-8') as pin_file:
pin_file.write(pinned_content)
except OSError:
print('EX: unable to write ' + pinned_filename)
def undo_pinned_post(base_dir: str, nickname: str, domain: str) -> None:
"""Removes pinned content for then given account
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
if not os.path.isfile(pinned_filename):
return
try:
os.remove(pinned_filename)
except OSError:
print('EX: undo_pinned_post unable to delete ' + pinned_filename)
def get_pinned_post_as_json(base_dir: str, http_prefix: str,
nickname: str, domain: str,
domain_full: str, system_language: str) -> {}:
"""Returns the pinned profile post as json
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
pinned_post_json = {}
actor = local_actor_url(http_prefix, nickname, domain_full)
if os.path.isfile(pinned_filename):
pinned_content = None
with open(pinned_filename, 'r', encoding='utf-8') as pin_file:
pinned_content = pin_file.read()
if pinned_content:
pinned_post_json = {
'atomUri': actor + '/pinned',
'attachment': [],
'attributedTo': actor,
'cc': [
actor + '/followers'
],
'content': pinned_content,
'contentMap': {
system_language: pinned_content
},
'id': actor + '/pinned',
'inReplyTo': None,
'inReplyToAtomUri': None,
'published': file_last_modified(pinned_filename),
'replies': {},
'sensitive': False,
'summary': None,
'tag': [],
'to': ['https://www.w3.org/ns/activitystreams#Public'],
'type': 'Note',
'url': replace_users_with_at(actor) + '/pinned'
}
return pinned_post_json
def json_pin_post(base_dir: str, http_prefix: str,
nickname: str, domain: str,
domain_full: str, system_language: str) -> {}:
"""Returns a pinned post as json
"""
pinned_post_json = \
get_pinned_post_as_json(base_dir, http_prefix,
nickname, domain,
domain_full, system_language)
items_list = []
if pinned_post_json:
items_list = [pinned_post_json]
actor = local_actor_url(http_prefix, nickname, domain_full)
post_context = get_individual_post_context()
return {
'@context': post_context,
'id': actor + '/collections/featured',
'orderedItems': items_list,
'totalItems': len(items_list),
'type': 'OrderedCollection'
}
def regenerate_index_for_box(base_dir: str,
nickname: str, domain: str,
box_name: str) -> None:
"""Generates an index for the given box if it doesn't exist
Used by unit tests to artificially create an index
"""
box_dir = acct_dir(base_dir, nickname, domain) + '/' + box_name
box_index_filename = box_dir + '.index'
if not os.path.isdir(box_dir):
return
if os.path.isfile(box_index_filename):
return
index_lines = []
for _, _, files in os.walk(box_dir):
for fname in files:
if ':##' not in fname:
continue
index_lines.append(fname)
break
index_lines.sort(reverse=True)
result = ''
try:
with open(box_index_filename, 'w+', encoding='utf-8') as fp_box:
for line in index_lines:
result += line + '\n'
fp_box.write(line + '\n')
except OSError:
print('EX: unable to generate index for ' + box_name + ' ' + result)
print('Index generated for ' + box_name + '\n' + result)
def create_public_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str,
in_reply_to_atom_uri: str, subject: str,
schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, is_article: bool, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str,
auto_cw_cache: {}) -> {}:
"""Public post
"""
domain_full = get_full_domain(domain, port)
is_moderation_report = False
event_uuid = None
category = None
join_mode = None
end_date = event_date
end_time = event_end_time
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
local_actor = local_actor_url(http_prefix, nickname, domain_full)
return _create_post_base(base_dir, nickname, domain, port,
'https://www.w3.org/ns/activitystreams#Public',
local_actor + '/followers',
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
event_uuid, category, join_mode,
end_date, end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate, buy_url,
chat_url, auto_cw_cache)
def create_reading_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
mentions_str: str, reading_update_type: str,
book_title: str, book_url: str, book_rating: float,
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str,
in_reply_to_atom_uri: str, subject: str,
schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, is_article: bool, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str,
auto_cw_cache: {}) -> {}:
""" reading status post
"""
content = ''
if mentions_str:
if not mentions_str.endswith(' '):
mentions_str += ' '
if reading_update_type == 'readingupdatewant':
content = mentions_str + translate['wants to read'] + \
' <a href="' + book_url + \
'"><i>' + book_title + '</i></a>'
elif reading_update_type == 'readingupdateread':
content = mentions_str + translate['reading'] + \
' <a href="' + book_url + \
'"><i>' + book_title + '</i></a>'
elif reading_update_type == 'readingupdatefinished':
content = mentions_str + translate['finished reading'] + \
' <a href="' + book_url + \
'"><i>' + book_title + '</i></a>'
elif reading_update_type == 'readingupdaterating' and book_rating > 0:
content = translate['rated'] + ' <a href="' + book_url + \
'"><i>' + book_title + '</i></a>'
if not content:
return None
post_json_object = \
create_public_post(base_dir,
nickname, domain, port, http_prefix,
content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript,
city, in_reply_to,
in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, event_end_time,
location, is_article, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate,
buy_url, chat_url, auto_cw_cache)
if post_json_object:
post_json_object['object']['tag'] = [{
'href': book_url,
'name': book_title,
'type': 'Edition'
}]
return post_json_object
def _append_citations_to_blog_post(base_dir: str,
nickname: str, domain: str,
blog_json: {}) -> None:
"""Appends any citations to a new blog post
"""
# append citations tags, stored in a file
citations_filename = \
acct_dir(base_dir, nickname, domain) + '/.citations.txt'
if not os.path.isfile(citations_filename):
return
citations_separator = '#####'
with open(citations_filename, 'r', encoding='utf-8') as fp_cit:
citations = fp_cit.readlines()
for line in citations:
if citations_separator not in line:
continue
sections = line.strip().split(citations_separator)
if len(sections) != 3:
continue
# date_str = sections[0]
title = sections[1]
link = sections[2]
tag_json = {
"type": "Article",
"name": title,
"url": link
}
blog_json['object']['tag'].append(tag_json)
def create_blog_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str, in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str) -> {}:
auto_cw_cache = {}
blog_json = \
create_public_post(base_dir,
nickname, domain, port, http_prefix,
content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, event_end_time, location,
True, system_language, conversation_id,
low_bandwidth, content_license_url,
media_license_url, media_creator,
languages_understood, translate, buy_url, chat_url,
auto_cw_cache)
url_str = get_url_from_post(blog_json['object']['url'])
obj_url = remove_html(url_str)
if '/@/' not in obj_url:
blog_json['object']['url'] = obj_url.replace('/@', '/users/')
_append_citations_to_blog_post(base_dir, nickname, domain, blog_json)
return blog_json
def create_news_post(base_dir: str,
domain: str, port: int, http_prefix: str,
content: str, save_to_file: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str, city: str,
subject: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str) -> {}:
auto_cw_cache = {}
client_to_server = False
in_reply_to = None
in_reply_to_atom_uri = None
schedule_post = False
event_date = None
event_time = None
event_end_time = None
location = None
blog = \
create_public_post(base_dir,
'news', domain, port, http_prefix,
content, save_to_file,
client_to_server, False,
attach_image_filename, media_type,
image_description, video_transcript, city,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, event_end_time, location,
True, system_language, conversation_id,
low_bandwidth, content_license_url,
media_license_url, media_creator,
languages_understood, translate, buy_url, chat_url,
auto_cw_cache)
blog['object']['type'] = 'Article'
return blog
def create_question_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, q_options: [],
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, subject: str, duration_days: int,
system_language: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
auto_cw_cache: {}) -> {}:
"""Question post with multiple choice options
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
buy_url = ''
chat_url = ''
is_moderation_report = False
is_article = False
in_reply_to = in_reply_to_atom_uri = None
schedule_post = False
event_date = event_time = location = event_uuid = category = None
join_mode = end_date = end_time = None
maximum_attendee_capacity = replies_moderation_option = None
anonymous_participation_enabled = event_status = ticket_url = None
conversation_id = None
message_json = \
_create_post_base(base_dir, nickname, domain, port,
'https://www.w3.org/ns/activitystreams#Public',
local_actor + '/followers',
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time,
location, event_uuid, category,
join_mode, end_date, end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled, event_status,
ticket_url, system_language,
conversation_id, low_bandwidth, content_license_url,
media_license_url, media_creator,
languages_understood, translate, buy_url,
chat_url, auto_cw_cache)
message_json['object']['type'] = 'Question'
message_json['object']['oneOf'] = []
message_json['object']['votersCount'] = 0
curr_time = date_utcnow()
days_since_epoch = \
int((curr_time - date_epoch()).days + duration_days)
end_time = date_epoch() + \
datetime.timedelta(days_since_epoch)
message_json['object']['endTime'] = end_time.strftime("%Y-%m-%dT%H:%M:%SZ")
for question_option in q_options:
message_json['object']['oneOf'].append({
"type": "Note",
"name": question_option,
"replies": {
"type": "Collection",
"totalItems": 0
}
})
return message_json
def create_unlisted_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
buy_url: str, chat_url: str,
auto_cw_cache: {}) -> {}:
"""Unlisted post. This has the #Public and followers links inverted.
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
is_moderation_report = False
is_article = False
event_uuid = category = join_mode = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
return _create_post_base(base_dir, nickname, domain, port,
local_actor + '/followers',
'https://www.w3.org/ns/activitystreams#Public',
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date,
event_time, location,
event_uuid, category, join_mode,
event_date, event_end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status,
ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate,
buy_url, chat_url, auto_cw_cache)
def create_followers_only_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str, content: str,
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str,
event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [],
translate: {}, buy_url: str,
chat_url: str,
auto_cw_cache: {}) -> {}:
"""Followers only post
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
is_moderation_report = False
is_article = False
event_uuid = category = join_mode = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
return _create_post_base(base_dir, nickname, domain, port,
local_actor + '/followers', None,
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
event_uuid, category, join_mode,
event_date, event_end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate,
buy_url, chat_url, auto_cw_cache)
def get_mentioned_people(base_dir: str, http_prefix: str,
content: str, domain: str, debug: bool) -> []:
"""Extracts a list of mentioned actors from the given message content
"""
if '@' not in content:
return None
mentions = []
words = content.split(' ')
for wrd in words:
if not wrd.startswith('@'):
continue
handle = wrd[1:]
if debug:
print('DEBUG: mentioned handle ' + handle)
if '@' not in handle:
handle = handle + '@' + domain
handle_dir = acct_handle_dir(base_dir, handle)
if not os.path.isdir(handle_dir):
continue
else:
external_domain = handle.split('@')[1]
if not ('.' in external_domain or
external_domain == 'localhost'):
continue
mentioned_nickname = handle.split('@')[0]
mentioned_domain = handle.split('@')[1].strip('\n').strip('\r')
if ':' in mentioned_domain:
mentioned_domain = remove_domain_port(mentioned_domain)
if not valid_nickname(mentioned_domain, mentioned_nickname):
continue
actor = \
local_actor_url(http_prefix, mentioned_nickname,
handle.split('@')[1])
mentions.append(actor)
return mentions
def create_direct_message_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str, content: str,
save_to_file: bool, client_to_server: bool,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, debug: bool,
schedule_post: bool,
event_date: str, event_time: str,
event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [],
dm_is_chat: bool, translate: {},
buy_url: str, chat_url: str,
auto_cw_cache: {}) -> {}:
"""Direct Message post
"""
content = resolve_petnames(base_dir, nickname, domain, content)
mentioned_people = \
get_mentioned_people(base_dir, http_prefix, content, domain, debug)
if debug:
print('mentioned_people: ' + str(mentioned_people))
if not mentioned_people:
return None
post_to = None
post_cc = None
is_moderation_report = False
is_article = False
event_uuid = category = join_mode = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
message_json = \
_create_post_base(base_dir, nickname, domain, port,
post_to, post_cc,
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
event_uuid, category, join_mode,
event_date, event_end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate, buy_url, chat_url,
auto_cw_cache)
# mentioned recipients go into To rather than Cc
message_json['to'] = message_json['object']['cc']
if not isinstance(message_json['to'], list):
message_json['to'] = [message_json['to']]
message_json['object']['to'] = message_json['to']
message_json['cc'] = []
message_json['object']['cc'] = []
if dm_is_chat:
message_json['object']['type'] = 'ChatMessage'
if schedule_post:
post_id = remove_id_ending(message_json['object']['id'])
save_post_to_box(base_dir, http_prefix, post_id,
nickname, domain, message_json, 'scheduled')
return message_json
def create_report_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, debug: bool, subject: str,
system_language: str, low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
languages_understood: [], translate: {},
auto_cw_cache: {}) -> {}:
"""Send a report to moderators
"""
domain_full = get_full_domain(domain, port)
# add a title to distinguish moderation reports from other posts
report_title = 'Moderation Report'
if not subject:
subject = report_title
else:
if not subject.startswith(report_title):
subject = report_title + ': ' + subject
# create the list of moderators from the moderators file
moderators_list = []
moderators_file = data_dir(base_dir) + '/moderators.txt'
if os.path.isfile(moderators_file):
with open(moderators_file, 'r', encoding='utf-8') as fp_mod:
for line in fp_mod:
line = line.strip('\n').strip('\r')
if line.startswith('#'):
continue
if line.startswith('/users/'):
line = line.replace('users', '')
if line.startswith('@'):
line = line[1:]
if '@' in line:
nick = line.split('@')[0]
moderator_actor = \
local_actor_url(http_prefix, nick, domain_full)
if moderator_actor not in moderators_list:
moderators_list.append(moderator_actor)
continue
if line.startswith('http') or \
line.startswith('ipfs') or \
line.startswith('ipns') or \
line.startswith('hyper'):
# must be a local address - no remote moderators
if '://' + domain_full + '/' in line:
if line not in moderators_list:
moderators_list.append(line)
else:
if '/' not in line:
moderator_actor = \
local_actor_url(http_prefix, line, domain_full)
if moderator_actor not in moderators_list:
moderators_list.append(moderator_actor)
if len(moderators_list) == 0:
# if there are no moderators then the admin becomes the moderator
admin_nickname = get_config_param(base_dir, 'admin')
if admin_nickname:
local_actor = \
local_actor_url(http_prefix, admin_nickname, domain_full)
moderators_list.append(local_actor)
if not moderators_list:
return None
if debug:
print('DEBUG: Sending report to moderators')
print(str(moderators_list))
post_to = moderators_list
post_cc = None
post_json_object = None
buy_url = ''
chat_url = ''
is_moderation_report = True
is_article = False
in_reply_to = in_reply_to_atom_uri = None
schedule_post = False
event_date = None
event_time = None
location = None
event_uuid = None
category = None
join_mode = None
end_date = None
end_time = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
conversation_id = None
for to_url in post_to:
# who is this report going to?
to_nickname = to_url.split('/users/')[1]
handle = to_nickname + '@' + domain
post_json_object = \
_create_post_base(base_dir, nickname, domain, port,
to_url, post_cc,
http_prefix, content, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time,
location, event_uuid, category,
join_mode, end_date, end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood, translate,
buy_url, chat_url, auto_cw_cache)
if not post_json_object:
continue
# save a notification file so that the moderator
# knows something new has appeared
new_report_file = acct_handle_dir(base_dir, handle) + '/.newReport'
if os.path.isfile(new_report_file):
continue
try:
with open(new_report_file, 'w+', encoding='utf-8') as fp_report:
fp_report.write(to_url + '/moderation')
except OSError:
print('EX: create_report_post unable to write ' + new_report_file)
return post_json_object
def _add_send_block(base_dir: str, nickname: str, domain: str,
inbox_url: str) -> None:
"""Adds an inbox which is blocking this instance
"""
send_block_filename = \
acct_dir(base_dir, nickname, domain) + '/send_blocks.txt'
inbox_url = inbox_url + '\n'
if inbox_url.endswith('/inbox\n'):
inbox_url = inbox_url.replace('/inbox\n', '\n')
if not os.path.isfile(send_block_filename):
try:
with open(send_block_filename, 'w+',
encoding='utf-8') as fp_blocks:
fp_blocks.write(inbox_url)
except OSError:
print('EX: _add_send_block unable to create ' +
send_block_filename)
return
if not text_in_file(inbox_url, send_block_filename, False):
try:
with open(send_block_filename, 'a+',
encoding='utf-8') as fp_blocks:
fp_blocks.write(inbox_url)
except OSError:
print('EX: _add_send_block unable to write ' +
send_block_filename)
def _remove_send_block(base_dir: str, nickname: str, domain: str,
inbox_url: str) -> None:
"""Removes an inbox which is blocking this instance
"""
send_block_filename = \
acct_dir(base_dir, nickname, domain) + '/send_blocks.txt'
inbox_url = inbox_url + '\n'
if inbox_url.endswith('/inbox\n'):
inbox_url = inbox_url.replace('/inbox\n', '\n')
if not os.path.isfile(send_block_filename):
return
if text_in_file(inbox_url, send_block_filename, False):
send_blocks_str = ''
try:
with open(send_block_filename, 'r',
encoding='utf-8') as fp_blocks:
send_blocks_str = fp_blocks.read()
except OSError:
print('EX: _remove_send_block unable to read ' +
send_block_filename)
send_blocks_str = send_blocks_str.replace(inbox_url, '')
try:
with open(send_block_filename, 'w+',
encoding='utf-8') as fp_blocks:
fp_blocks.write(send_blocks_str)
except OSError:
print('EX: _remove_send_block unable to write ' +
send_block_filename)
def thread_send_post(session, post_json_str: str, federation_list: [],
inbox_url: str, base_dir: str,
signature_header_json: {},
signature_header_json_ld: {},
post_log: [], debug: bool,
http_prefix: str, domain_full: str,
nickname: str, domain: str) -> None:
"""Sends a with retries
"""
tries = 0
send_interval_sec = 30
for _ in range(20):
post_result = None
unauthorized = False
if debug:
print('Getting post_json_string for ' + inbox_url)
try:
post_result, unauthorized, return_code = \
post_json_string(session, post_json_str, federation_list,
inbox_url, signature_header_json,
debug, http_prefix, domain_full)
if return_code in range(500, 600):
# if an instance is returning a code which indicates that
# it might have a runtime error, like 503, then don't
# continue to post to it
break
if debug:
print('Obtained post_json_string for ' + inbox_url +
' unauthorized: ' + str(unauthorized))
except BaseException as ex:
print('ERROR: post_json_string failed ' + str(ex))
if unauthorized:
# try again with application/ld+json header
post_result = None
unauthorized = False
if debug:
print('Getting ld post_json_string for ' + inbox_url)
try:
post_result, unauthorized, return_code = \
post_json_string(session, post_json_str, federation_list,
inbox_url, signature_header_json_ld,
debug, http_prefix, domain_full)
if return_code in range(500, 600):
# if an instance is returning a code which indicates that
# it might have a runtime error, like 503, then don't
# continue to post to it
break
if debug:
print('Obtained ld post_json_string for ' + inbox_url +
' unauthorized: ' + str(unauthorized))
except BaseException as ex:
print('ERROR: ld post_json_string failed ' + str(ex))
if unauthorized:
_add_send_block(base_dir, nickname, domain, inbox_url)
if debug:
print('WARN: thread_send_post: Post is unauthorized ' +
inbox_url + ' ' + post_json_str)
else:
print('WARN: thread_send_post: Post is unauthorized ' +
inbox_url)
break
if post_result:
log_str = 'Success on try ' + str(tries) + ': ' + post_json_str
else:
log_str = 'Retry ' + str(tries) + ': ' + post_json_str
post_log.append(log_str)
# keep the length of the log finite
# Don't accumulate massive files on systems with limited resources
while len(post_log) > 16:
post_log.pop(0)
if debug:
# save the log file
post_log_filename = base_dir + '/post.log'
if os.path.isfile(post_log_filename):
try:
with open(post_log_filename, 'a+',
encoding='utf-8') as log_file:
log_file.write(log_str + '\n')
except OSError:
print('EX: thread_send_post unable to append ' +
post_log_filename)
else:
try:
with open(post_log_filename, 'w+',
encoding='utf-8') as log_file:
log_file.write(log_str + '\n')
except OSError:
print('EX: thread_send_post unable to write ' +
post_log_filename)
if post_result:
_remove_send_block(base_dir, nickname, domain, inbox_url)
if debug:
print('DEBUG: successful json post to ' + inbox_url)
# our work here is done
break
if debug:
print(post_json_str)
print('DEBUG: json post to ' + inbox_url +
' failed. Waiting for ' +
str(send_interval_sec) + ' seconds.')
time.sleep(send_interval_sec)
tries += 1
def _get_destination_inbox_domain(inbox_url: str, to_domain: str) -> str:
""" the inbox domain being sent to isn't always the same
as the destination instance domain. eg public-api.wordpress.com
rather than the wordpress instance domain
"""
inbox_url_domain, inbox_url_port = get_domain_from_actor(inbox_url)
if not inbox_url_domain:
return inbox_url
inbox_domain = get_full_domain(inbox_url_domain, inbox_url_port)
if inbox_domain != to_domain:
print('_get_destination_inbox_domain ' +
'inbox domain is not the same as instance domain ' +
inbox_domain + ' != ' + to_domain)
return inbox_domain
return to_domain
def send_post(signing_priv_key_pem: str, project_version: str,
session, base_dir: str, nickname: str, domain: str, port: int,
to_nickname: str, to_domain: str, to_port: int, cc_str: str,
http_prefix: str, content: str,
save_to_file: bool, client_to_server: bool,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str, city: str,
federation_list: [], send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
is_article: bool, system_language: str,
languages_understood: [],
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
low_bandwidth: bool, content_license_url: str,
media_license_url: str, media_creator: str,
translate: {}, buy_url: str, chat_url: str,
auto_cw_cache: {},
debug: bool, in_reply_to: str,
in_reply_to_atom_uri: str, subject: str) -> int:
"""Post to another inbox. Used by unit tests.
"""
with_digest = True
conversation_id = None
if to_nickname == 'inbox':
# shared inbox actor on @domain@domain
to_nickname = to_domain
to_domain = get_full_domain(to_domain, to_port)
handle = http_prefix + '://' + to_domain + '/@' + to_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return 1
if not isinstance(wf_request, dict):
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return 1
if not client_to_server:
post_to_box = 'inbox'
else:
post_to_box = 'outbox'
if is_article:
post_to_box = 'tlblogs'
# get the actor inbox for the To handle
origin_domain = domain
(inbox_url, _, pub_key, to_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, post_to_box,
72533, system_language)
if not inbox_url:
return 3
if not pub_key:
return 4
if not to_person_id:
return 5
# shared_inbox is optional
is_moderation_report = False
schedule_post = False
event_date = event_time = location = None
event_uuid = category = None
join_mode = None
end_date = None
end_time = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = ticket_url = None
post_json_object = \
_create_post_base(base_dir, nickname, domain, port,
to_person_id, cc_str, http_prefix, content,
save_to_file, client_to_server,
comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article, in_reply_to,
in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, location,
event_uuid, category,
join_mode,
end_date, end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood,
translate, buy_url, chat_url,
auto_cw_cache)
# get the senders private key
private_key_pem = get_person_key(nickname, domain, base_dir,
'private', debug)
if len(private_key_pem) == 0:
return 6
# NOTE: the inbox domain being sent to isn't always the same
# as the destination instance domain. eg public-api.wordpress.com
# rather than the wordpress instance domain
to_domain = _get_destination_inbox_domain(inbox_url, to_domain)
if to_domain not in inbox_url:
return 7
# remove the domain to leave the path on its own
post_path = inbox_url.split(to_domain, 1)[1]
if not post_json_object.get('signature'):
try:
signed_post_json_object = post_json_object.copy()
generate_json_signature(signed_post_json_object, private_key_pem)
post_json_object = signed_post_json_object
except BaseException as ex:
print('WARN: send_post failed to JSON-LD sign post, ' + str(ex))
pprint(signed_post_json_object)
# convert json to string so that there are no
# subsequent conversions after creating message body digest
post_json_str = json.dumps(post_json_object)
# construct the http header, including the message body digest
signature_header_json = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str, 'application/activity+json')
signature_header_json_ld = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str, 'application/ld+json')
# if the "to" domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
domain_full = get_full_domain(domain, port)
if to_domain in shared_items_federated_domains:
if shared_item_federation_tokens.get(domain_full):
signature_header_json['Origin'] = domain_full
signature_header_json_ld['Origin'] = domain_full
signature_header_json['SharesCatalog'] = \
shared_item_federation_tokens[domain_full]
signature_header_json_ld['SharesCatalog'] = \
shared_item_federation_tokens[domain_full]
if debug:
print('SharesCatalog added to header')
elif debug:
print(domain_full + ' not in shared_item_federation_tokens')
elif debug:
print(to_domain + ' not in shared_items_federated_domains ' +
str(shared_items_federated_domains))
if debug:
print('signature_header_json: ' + str(signature_header_json))
# Keep the number of threads being used small
while len(send_threads) > 1000:
print('WARN: Maximum threads reached - killing send thread')
send_threads[0].kill()
send_threads.pop(0)
print('WARN: thread killed')
print('THREAD: thread_send_post')
thr = \
thread_with_trace(target=thread_send_post,
args=(session,
post_json_str,
federation_list,
inbox_url, base_dir,
signature_header_json.copy(),
signature_header_json_ld.copy(),
post_log, debug, http_prefix,
domain_full,
nickname, domain), daemon=True)
send_threads.append(thr)
begin_thread(thr, 'send_post')
return 0
def send_post_via_server(signing_priv_key_pem: str, project_version: str,
base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
to_nickname: str, to_domain: str, to_port: int,
cc_str: str,
http_prefix: str, content: str,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, video_transcript: str,
city: str, cached_webfingers: {}, person_cache: {},
is_article: bool, system_language: str,
languages_understood: [],
low_bandwidth: bool,
content_license_url: str,
media_license_url: str, media_creator: str,
event_date: str, event_time: str, event_end_time: str,
location: str, translate: {},
buy_url: str, chat_url: str, auto_cw_cache: {},
debug: bool, in_reply_to: str,
in_reply_to_atom_uri: str,
conversation_id: str, subject: str) -> int:
"""Send a post via a proxy (c2s)
"""
if not session:
print('WARN: No session for send_post_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
from_domain_full, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: post webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: post webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
if is_article:
post_to_box = 'tlblogs'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain_full, post_to_box,
82796, system_language)
if not inbox_url:
if debug:
print('DEBUG: post no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: post no actor was found for ' + handle)
return 4
# Get the json for the c2s post, not saving anything to file
# Note that base_dir is set to None
save_to_file = False
client_to_server = True
if to_domain.lower().endswith('public'):
to_person_id = 'https://www.w3.org/ns/activitystreams#Public'
cc_str = \
local_actor_url(http_prefix, from_nickname, from_domain_full) + \
'/followers'
else:
if to_domain.lower().endswith('followers') or \
to_domain.lower().endswith('followersonly'):
to_person_id = \
local_actor_url(http_prefix,
from_nickname, from_domain_full) + \
'/followers'
else:
to_domain_full = get_full_domain(to_domain, to_port)
to_person_id = \
local_actor_url(http_prefix, to_nickname, to_domain_full)
is_moderation_report = False
schedule_post = False
event_uuid = category = join_mode = None
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = ticket_url = None
post_json_object = \
_create_post_base(base_dir,
from_nickname, from_domain, from_port,
to_person_id, cc_str, http_prefix, content,
save_to_file, client_to_server,
comments_enabled,
attach_image_filename, media_type,
image_description, video_transcript, city,
is_moderation_report, is_article, in_reply_to,
in_reply_to_atom_uri, subject, schedule_post,
event_date, event_time, location,
event_uuid, category, join_mode,
event_date, event_end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
media_license_url, media_creator,
languages_understood,
translate, buy_url, chat_url, auto_cw_cache)
auth_header = create_basic_auth_header(from_nickname, password)
if attach_image_filename:
headers = {
'host': from_domain_full,
'Authorization': auth_header
}
post_result = \
post_image(session, attach_image_filename, [],
inbox_url, headers, http_prefix, from_domain_full)
if not post_result:
if debug:
print('DEBUG: post failed to upload image')
# return 9
headers = {
'host': from_domain_full,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_dumps = json.dumps(post_json_object)
post_result, unauthorized, return_code = \
post_json_string(session, post_dumps, [],
inbox_url, headers, debug,
http_prefix, from_domain_full,
5, True)
if not post_result:
if debug:
if unauthorized:
print('DEBUG: POST failed for c2s to ' +
inbox_url + ' unathorized')
else:
print('DEBUG: POST failed for c2s to ' +
inbox_url + ' return code ' + str(return_code))
return 5
if debug:
print('DEBUG: c2s POST success')
return 0
def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}:
"""Returns a dictionary with followers grouped by domain
"""
handle = nickname + '@' + domain
followers_filename = acct_handle_dir(base_dir, handle) + '/followers.txt'
if not os.path.isfile(followers_filename):
return None
grouped = {}
with open(followers_filename, 'r', encoding='utf-8') as foll_file:
for follower_handle in foll_file:
if '@' not in follower_handle:
continue
fhandle1 = follower_handle.strip()
fhandle = remove_eol(fhandle1)
follower_domain = fhandle.split('@')[1]
if not grouped.get(follower_domain):
grouped[follower_domain] = [fhandle]
else:
grouped[follower_domain].append(fhandle)
return grouped
def _add_followers_to_public_post(post_json_object: {}) -> None:
"""Adds followers entry to cc if it doesn't exist
"""
if not post_json_object.get('actor'):
return
if isinstance(post_json_object['object'], str):
if not post_json_object.get('to'):
return
if len(post_json_object['to']) > 1:
return
if len(post_json_object['to']) == 0:
return
if not post_json_object['to'][0].endswith('#Public'):
if not post_json_object['to'][0] == 'as:Public':
if not post_json_object['to'][0] == 'Public':
return
if post_json_object.get('cc'):
return
actor_url = get_actor_from_post(post_json_object)
post_json_object['cc'] = [actor_url + '/followers']
elif has_object_dict(post_json_object):
if not post_json_object['object'].get('to'):
return
if len(post_json_object['object']['to']) > 1:
return
if len(post_json_object['object']['to']) == 0:
return
if not post_json_object['object']['to'][0].endswith('#Public'):
if not post_json_object['object']['to'][0] == 'as:Public':
if not post_json_object['object']['to'][0] == 'Public':
return
if post_json_object['object'].get('cc'):
return
actor_url = get_actor_from_post(post_json_object)
post_json_object['object']['cc'] = \
[actor_url + '/followers']
def send_signed_json(post_json_object: {}, session, base_dir: str,
nickname: str, domain: str, port: int,
to_nickname: str, to_domain: str,
to_port: int, http_prefix: str,
client_to_server: bool, federation_list: [],
send_threads: [], post_log: [], cached_webfingers: {},
person_cache: {}, debug: bool, project_version: str,
shared_items_token: str, group_account: bool,
signing_priv_key_pem: str,
source_id: int, curr_domain: str,
onion_domain: str, i2p_domain: str,
extra_headers: {}, sites_unavailable: [],
system_language: str) -> int:
"""Sends a signed json object to an inbox/outbox
"""
if debug:
print('DEBUG: send_signed_json start')
if not session:
print('WARN: No session specified for send_signed_json')
return 8
with_digest = True
if to_domain.endswith('.onion') or to_domain.endswith('.i2p'):
http_prefix = 'http'
if to_nickname == 'inbox':
# shared inbox actor on @domain@domain
to_nickname = to_domain
to_domain = get_full_domain(to_domain, to_port)
to_domain_url = http_prefix + '://' + to_domain
if not site_is_active(to_domain_url, 10, sites_unavailable):
print('send_signed_json domain is inactive: ' + to_domain_url)
return 9
print('Domain is active: ' + to_domain_url)
handle_base = to_domain_url + '/@'
if to_nickname:
handle = handle_base + to_nickname
else:
single_user_instance_nickname = 'dev'
handle = handle_base + single_user_instance_nickname
if debug:
print('DEBUG: send_signed_json handle - ' + handle +
' to_port ' + str(to_port))
# domain shown in the user agent
ua_domain = curr_domain
if to_domain.endswith('.onion'):
ua_domain = onion_domain
elif to_domain.endswith('.i2p'):
ua_domain = i2p_domain
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
ua_domain, project_version, debug,
group_account, signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: send_signed_json webfinger for ' +
handle + ' failed')
return 1
if not isinstance(wf_request, dict):
print('WARN: send_signed_json webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
if wf_request.get('errors'):
if debug:
print('DEBUG: send_signed_json webfinger for ' + handle +
' failed with errors ' + str(wf_request['errors']))
if not client_to_server:
post_to_box = 'inbox'
else:
post_to_box = 'outbox'
# get the actor inbox/outbox for the To handle
origin_domain = domain
(inbox_url, _, pub_key, to_person_id, shared_inbox_url, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, post_to_box,
source_id, system_language)
print("send_signed_json inbox_url: " + str(inbox_url))
print("send_signed_json to_person_id: " + str(to_person_id))
print("send_signed_json shared_inbox_url: " + str(shared_inbox_url))
if inbox_url:
if inbox_url.endswith('/actor/inbox') or \
inbox_url.endswith('/instance.actor/inbox'):
inbox_url = shared_inbox_url
if not inbox_url:
if debug:
print('DEBUG: send_signed_json missing inbox_url')
return 3
if debug:
print('DEBUG: send_signed_json sending to endpoint ' + inbox_url)
if not pub_key:
if debug:
print('DEBUG: send_signed_json missing pubkey')
return 4
if not to_person_id:
if debug:
print('DEBUG: send_signed_json missing person_id')
return 5
# shared_inbox is optional
# get the senders private key
account_domain = origin_domain
if onion_domain:
if account_domain == onion_domain:
account_domain = curr_domain
if i2p_domain:
if account_domain == i2p_domain:
account_domain = curr_domain
private_key_pem = \
get_person_key(nickname, account_domain, base_dir, 'private', debug)
if len(private_key_pem) == 0:
if debug:
print('DEBUG: send_signed_json private key not found for ' +
nickname + '@' + account_domain +
' in ' + base_dir + '/keys/private')
return 6
# NOTE: the inbox domain being sent to isn't always the same
# as the destination instance domain. eg public-api.wordpress.com
# rather than the wordpress instance domain
to_domain = _get_destination_inbox_domain(inbox_url, to_domain)
if to_domain not in inbox_url:
if debug:
print('DEBUG: send_signed_json ' +
to_domain + ' is not in ' + inbox_url)
return 7
# remove the domain to leave the path on its own
post_path = inbox_url.split(to_domain, 1)[1]
_add_followers_to_public_post(post_json_object)
if not post_json_object.get('signature'):
try:
signed_post_json_object = post_json_object.copy()
generate_json_signature(signed_post_json_object, private_key_pem)
post_json_object = signed_post_json_object
except BaseException as ex:
print('WARN: send_signed_json failed to JSON-LD sign post, ' +
str(ex))
pprint(signed_post_json_object)
# convert json to string so that there are no
# subsequent conversions after creating message body digest
post_json_str = json.dumps(post_json_object)
# if the sender domain has changed from clearnet to onion or i2p
# then change the content of the post accordingly
if debug:
print('send_signed_json checking for changed origin domain: ' +
domain + ' ' + curr_domain)
if domain != curr_domain:
if not curr_domain.endswith('.onion') and \
not curr_domain.endswith('.i2p'):
if debug:
print('send_signed_json ' +
'changing post content sender domain from ' +
curr_domain + ' to ' + domain)
post_json_str = \
post_json_str.replace(curr_domain, domain)
# construct the http header, including the message body digest
signature_header_json = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str,
'application/activity+json')
signature_header_json_ld = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str,
'application/ld+json')
# optionally add a token so that the receiving instance may access
# your shared items catalog
if shared_items_token:
signature_header_json['Origin'] = get_full_domain(domain, port)
signature_header_json['SharesCatalog'] = shared_items_token
elif debug:
print('send_signed_json not sending shared items federation token')
# add any extra headers
for header_title, header_text in extra_headers.items():
signature_header_json[header_title] = header_text
# Keep the number of threads being used small
while len(send_threads) > 1000:
print('WARN: send_signed_json maximum threads reached - ' +
'killing send thread')
send_threads[0].kill()
send_threads.pop(0)
print('WARN: send_signed_json thread killed')
if debug:
print('DEBUG: send_signed_json starting thread to send post')
pprint(post_json_object)
domain_full = get_full_domain(domain, port)
print('THREAD: send_signed_json thread_send_post 2')
thr = \
thread_with_trace(target=thread_send_post,
args=(session,
post_json_str,
federation_list,
inbox_url, base_dir,
signature_header_json.copy(),
signature_header_json_ld.copy(),
post_log, debug,
http_prefix, domain_full,
nickname, domain), daemon=True)
send_threads.append(thr)
# begin_thread(thr, 'send_signed_json')
return 0
def add_to_field(activity_type: str, post_json_object: {},
debug: bool) -> ({}, bool):
"""The Follow/Add/Remove activity doesn't have a 'to' field and so one
needs to be added so that activity distribution happens in a consistent way
Returns true if a 'to' field exists or was added
"""
if post_json_object.get('to'):
return post_json_object, True
if debug:
pprint(post_json_object)
print('DEBUG: no "to" field when sending to named addresses 2')
is_same_type = False
to_field_added = False
if post_json_object.get('object'):
if isinstance(post_json_object['object'], str):
if post_json_object.get('type'):
if post_json_object['type'] == activity_type:
is_same_type = True
if debug:
print('DEBUG: ' +
'add_to_field1 "to" field assigned to ' +
activity_type)
to_address = post_json_object['object']
if '/statuses/' in to_address:
to_address = to_address.split('/statuses/')[0]
post_json_object['to'] = [to_address]
if debug:
print('DEBUG: "to" address added: ' + to_address)
to_field_added = True
elif has_object_dict(post_json_object):
# add a to field to bookmark add or remove
if post_json_object.get('type') and \
post_json_object.get('actor') and \
post_json_object['object'].get('type'):
if post_json_object['type'] == 'Add' or \
post_json_object['type'] == 'Remove':
if post_json_object['object']['type'] == 'Document':
actor_url = get_actor_from_post(post_json_object)
post_json_object['to'] = [actor_url]
post_json_object['object']['to'] = [actor_url]
to_field_added = True
if not to_field_added and \
post_json_object['object'].get('type'):
if post_json_object['object']['type'] == activity_type:
is_same_type = True
if isinstance(post_json_object['object']['object'], str):
if debug:
print('DEBUG: add_to_field2 ' +
'"to" field assigned to ' +
activity_type)
to_address = post_json_object['object']['object']
if '/statuses/' in to_address:
to_address = to_address.split('/statuses/')[0]
post_json_object['object']['to'] = [to_address]
post_json_object['to'] = [to_address]
if debug:
print('DEBUG: "to" address added: ' + to_address)
to_field_added = True
if not is_same_type:
return post_json_object, True
if to_field_added:
return post_json_object, True
return post_json_object, False
def _is_profile_update(post_json_object: {}) -> bool:
"""Is the given post a profile update?
for actor updates there is no 'to' within the object
"""
if post_json_object.get('type'):
if has_object_string_type(post_json_object, False):
if (post_json_object['type'] == 'Update' and
(post_json_object['object']['type'] == 'Person' or
post_json_object['object']['type'] == 'Application' or
post_json_object['object']['type'] == 'Group' or
post_json_object['object']['type'] == 'Service')):
return True
return False
def _send_to_named_addresses(server, session, session_onion, session_i2p,
base_dir: str,
nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {},
sites_unavailable: [],
system_language: str) -> None:
"""sends a post to the specific named addresses in to/cc
"""
if not session:
print('WARN: No session for sendToNamedAddresses')
return
if not post_json_object.get('object'):
return
is_profile_update = False
if has_object_dict(post_json_object):
if _is_profile_update(post_json_object):
# use the original object, which has a 'to'
recipients_object = post_json_object
is_profile_update = True
if not is_profile_update:
if not post_json_object['object'].get('to'):
if debug:
pprint(post_json_object)
print('DEBUG: ' +
'no "to" field when sending to named addresses')
if has_object_string_type(post_json_object, debug):
if post_json_object['object']['type'] == 'Follow' or \
post_json_object['object']['type'] == 'Join':
post_json_obj2 = post_json_object['object']['object']
if isinstance(post_json_obj2, str):
if debug:
print('DEBUG: _send_to_named_addresses ' +
'"to" field assigned to Follow')
post_json_object['object']['to'] = \
[post_json_object['object']['object']]
if not post_json_object['object'].get('to'):
return
recipients_object = post_json_object['object']
else:
post_json_object, field_added = \
add_to_field('Follow', post_json_object, debug)
if not field_added:
return
post_json_object, field_added = \
add_to_field('Like', post_json_object, debug)
if not field_added:
return
recipients_object = post_json_object
recipients = []
recipient_type = ('to', 'cc')
for rtype in recipient_type:
if not recipients_object.get(rtype):
continue
if isinstance(recipients_object[rtype], list):
if debug:
pprint(recipients_object)
print('recipients_object: ' + str(recipients_object[rtype]))
for address in recipients_object[rtype]:
if not address:
continue
if '/' not in address:
continue
if address.endswith('#Public') or \
address == 'as:Public' or \
address == 'Public':
continue
if address.endswith('/followers'):
continue
recipients.append(address)
elif isinstance(recipients_object[rtype], str):
address = recipients_object[rtype]
if address:
if '/' in address:
if address.endswith('#Public') or \
address == 'as:Public' or \
address == 'Public':
continue
if address.endswith('/followers'):
continue
recipients.append(address)
if not recipients:
if debug:
print('DEBUG: no individual recipients')
return
if debug:
print('DEBUG: Sending individually addressed posts: ' +
str(recipients))
domain_full = get_full_domain(domain, port)
# randomize the recipients list order, so that we are not favoring
# any particular account in terms of delivery time
random.shuffle(recipients)
# this is after the message has arrived at the server
client_to_server = False
for address in recipients:
to_nickname = get_nickname_from_actor(address)
if not to_nickname:
continue
to_domain, to_port = get_domain_from_actor(address)
if not to_domain:
continue
to_domain_full = get_full_domain(to_domain, to_port)
# Don't send profile/actor updates to yourself
if is_profile_update:
if nickname == to_nickname and \
domain_full == to_domain_full:
if debug:
print('Not sending profile update to self. ' +
nickname + '@' + domain_full)
continue
# if we have an alt onion domain and we are sending to
# another onion domain then switch the clearnet
# domain for the onion one
from_domain = domain
from_domain_full = get_full_domain(domain, port)
from_http_prefix = http_prefix
curr_session = session
curr_proxy_type = proxy_type
session_type = 'default'
if onion_domain:
if not from_domain.endswith('.onion') and \
to_domain.endswith('.onion'):
from_domain = onion_domain
from_domain_full = onion_domain
from_http_prefix = 'http'
curr_session = session_onion
port = 80
to_port = 80
curr_proxy_type = 'tor'
session_type = 'tor'
if i2p_domain:
if not from_domain.endswith('.i2p') and \
to_domain.endswith('.i2p'):
from_domain = i2p_domain
from_domain_full = i2p_domain
from_http_prefix = 'http'
curr_session = session_i2p
port = 80
to_port = 80
curr_proxy_type = 'i2p'
session_type = 'i2p'
extra_headers = {}
# followers synchronization header
# See https://github.com/mastodon/mastodon/pull/14510
# https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-8fcf.md
sending_actor = \
from_http_prefix + '://' + from_domain_full + '/users/' + nickname
_, followers_sync_hash = \
update_followers_sync_cache(base_dir,
nickname, domain,
http_prefix, domain_full,
to_domain_full,
followers_sync_cache)
if followers_sync_hash:
collection_sync_str = \
'collectionId="' + sending_actor + '/followers", ' + \
'url="' + sending_actor + '/followers_synchronization", ' + \
'digest="' + followers_sync_hash + '"'
extra_headers["Collection-Synchronization"] = collection_sync_str
if debug:
print('DEBUG: extra_headers, ' + str(extra_headers))
if debug:
to_domain_full = get_full_domain(to_domain, to_port)
print('DEBUG: Post sending s2s: ' +
nickname + '@' + from_domain_full +
' to ' + to_nickname + '@' + to_domain_full)
# if the "to" domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
shared_items_token = None
if to_domain in shared_items_federated_domains:
if shared_item_federation_tokens.get(from_domain_full):
shared_items_token = \
shared_item_federation_tokens[from_domain_full]
group_account = has_group_type(base_dir, address, person_cache)
if not curr_session:
curr_session = create_session(curr_proxy_type)
if server:
if session_type == 'tor':
server.session_onion = curr_session
elif session_type == 'i2p':
server.session_i2p = curr_session
else:
server.session = curr_session
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
from_http_prefix, client_to_server,
federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 34436782,
domain, onion_domain, i2p_domain,
extra_headers, sites_unavailable,
system_language)
def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str,
i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {},
sites_unavailable: [],
system_language: str):
"""Returns a thread used to send a post to named addresses
"""
print('THREAD: _send_to_named_addresses')
send_thread = \
thread_with_trace(target=_send_to_named_addresses,
args=(server, session, session_onion, session_i2p,
base_dir, nickname, domain,
onion_domain, i2p_domain, port,
http_prefix, federation_list,
send_threads, post_log,
cached_webfingers, person_cache,
post_json_object, debug,
project_version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem,
proxy_type,
followers_sync_cache,
sites_unavailable,
system_language), daemon=True)
if not begin_thread(send_thread, 'send_to_named_addresses_thread'):
print('WARN: socket error while starting ' +
'thread to send to named addresses.')
return None
return send_thread
def _has_shared_inbox(session, http_prefix: str, domain: str,
debug: bool, signing_priv_key_pem: str,
ua_domain: str) -> bool:
"""Returns true if the given domain has a shared inbox
This tries the new and the old way of webfingering the shared inbox
"""
try_handles = []
if ':' not in domain:
try_handles.append(domain + '@' + domain)
try_handles.append('inbox@' + domain)
for handle in try_handles:
wf_request = webfinger_handle(session, handle, http_prefix, {},
ua_domain, __version__, debug, False,
signing_priv_key_pem)
if wf_request:
if isinstance(wf_request, dict):
if not wf_request.get('errors'):
return True
return False
def _sending_profile_update(post_json_object: {}) -> bool:
"""Returns true if the given json is a profile update
"""
if post_json_object['type'] != 'Update':
return False
if not has_object_string_type(post_json_object, False):
return False
activity_type = post_json_object['object']['type']
if activity_type in ('Person', 'Application', 'Group', 'Service'):
return True
return False
def send_to_followers(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
sites_unavailable: [],
system_language: str) -> None:
"""sends a post to the followers of the given nickname
"""
print('send_to_followers')
if not _post_is_addressed_to_followers(nickname, domain,
port, http_prefix,
post_json_object):
if debug:
print('Post is not addressed to followers')
return
print('Post is addressed to followers')
extra_headers = {}
grouped = group_followers_by_domain(base_dir, nickname, domain)
if not grouped:
if debug:
print('Post to followers did not resolve any domains')
return
print('Post to followers resolved domains')
# print(str(grouped))
# this is after the message has arrived at the server
client_to_server = False
curr_proxy_type = None
if domain.endswith('.onion'):
curr_proxy_type = 'tor'
elif domain.endswith('.i2p'):
curr_proxy_type = 'i2p'
sending_start_time = date_utcnow()
print('Sending post to followers begins ' +
sending_start_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
sending_ctr = 0
# randomize the order of sending to instances
randomized_instances = []
for follower_domain, follower_handles in grouped.items():
randomized_instances.append([follower_domain, follower_handles])
random.shuffle(randomized_instances)
# send out to each instance
for group_send in randomized_instances:
follower_domain = group_send[0]
follower_handles = group_send[1]
print('Sending post to followers progress ' +
str(int(sending_ctr * 100 / len(grouped.items()))) + '% ' +
follower_domain)
sending_ctr += 1
if debug:
pprint(follower_handles)
# if the followers domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
shared_items_token = None
if follower_domain in shared_items_federated_domains:
domain_full = get_full_domain(domain, port)
if shared_item_federation_tokens.get(domain_full):
shared_items_token = shared_item_federation_tokens[domain_full]
# check that the follower's domain is active
follower_domain_url = http_prefix + '://' + follower_domain
if not site_is_active(follower_domain_url, 10, sites_unavailable):
print('Sending post to followers domain is inactive: ' +
follower_domain_url)
continue
print('Sending post to followers domain is active: ' +
follower_domain_url)
# select the appropriate session
curr_session = session
curr_http_prefix = http_prefix
if onion_domain:
if follower_domain.endswith('.onion'):
curr_session = session_onion
curr_http_prefix = 'http'
if i2p_domain:
if follower_domain.endswith('.i2p'):
curr_session = session_i2p
curr_http_prefix = 'http'
# get the domain showin by the user agent
ua_domain = domain
if follower_domain.endswith('.onion'):
ua_domain = onion_domain
elif follower_domain.endswith('.i2p'):
ua_domain = i2p_domain
with_shared_inbox = \
_has_shared_inbox(curr_session, curr_http_prefix, follower_domain,
debug, signing_priv_key_pem, ua_domain)
if debug:
if with_shared_inbox:
print(follower_domain + ' has shared inbox')
if not with_shared_inbox:
print('Sending post to followers, ' + follower_domain +
' does not have a shared inbox')
to_port = port
index = 0
to_domain = follower_handles[index].split('@')[1]
if ':' in to_domain:
to_port = get_port_from_domain(to_domain)
to_domain = remove_domain_port(to_domain)
# if we are sending to an onion domain and we
# have an alt onion domain then use the alt
from_domain = domain
from_http_prefix = http_prefix
session_type = 'default'
if onion_domain:
if to_domain.endswith('.onion'):
from_domain = onion_domain
from_http_prefix = 'http'
port = 80
to_port = 80
curr_proxy_type = 'tor'
session_type = 'tor'
if i2p_domain:
if to_domain.endswith('.i2p'):
from_domain = i2p_domain
from_http_prefix = 'http'
port = 80
to_port = 80
curr_proxy_type = 'i2p'
session_type = 'i2p'
if not curr_session:
curr_session = create_session(curr_proxy_type)
if server:
if session_type == 'tor':
server.session_onion = curr_session
elif session_type == 'i2p':
server.session_i2p = curr_session
else:
server.session = curr_session
if with_shared_inbox:
to_nickname = follower_handles[index].split('@')[0]
group_account = False
if to_nickname.startswith('!'):
group_account = True
to_nickname = to_nickname[1:]
# if there are more than one followers on the domain
# then send the post to the shared inbox
if len(follower_handles) > 1:
to_nickname = 'inbox'
if to_nickname != 'inbox' and post_json_object.get('type'):
if _sending_profile_update(post_json_object):
print('Sending post to followers ' +
'shared inbox of ' + to_domain)
to_nickname = 'inbox'
print('Sending post to followers from ' +
nickname + '@' + domain +
' to ' + to_nickname + '@' + to_domain)
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
from_http_prefix,
client_to_server, federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 639342,
domain, onion_domain, i2p_domain,
extra_headers, sites_unavailable,
system_language)
else:
# randomize the order of handles, so that we are not
# favoring any particular account in terms of its delivery time
random.shuffle(follower_handles)
# send to individual followers without using a shared inbox
for handle in follower_handles:
print('Sending post to followers ' + handle)
to_nickname = handle.split('@')[0]
group_account = False
if to_nickname.startswith('!'):
group_account = True
to_nickname = to_nickname[1:]
if post_json_object['type'] != 'Update':
print('Sending post to followers from ' +
nickname + '@' + domain + ' to ' +
to_nickname + '@' + to_domain)
else:
print('Sending post to followers profile update from ' +
nickname + '@' + domain + ' to ' +
to_nickname + '@' + to_domain)
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
from_http_prefix,
client_to_server, federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 634219,
domain, onion_domain, i2p_domain,
extra_headers, sites_unavailable,
system_language)
time.sleep(4)
if debug:
print('DEBUG: End of send_to_followers')
sending_end_time = date_utcnow()
sending_mins = \
int((sending_end_time - sending_start_time).total_seconds() / 60)
print('Sending post to followers ends ' + str(sending_mins) + ' mins')
def send_to_followers_thread(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
sites_unavailable: [],
system_language: str):
"""Returns a thread used to send a post to followers
"""
print('THREAD: send_to_followers')
send_thread = \
thread_with_trace(target=send_to_followers,
args=(server, session, session_onion, session_i2p,
base_dir, nickname, domain,
onion_domain, i2p_domain, port,
http_prefix, federation_list,
send_threads, post_log,
cached_webfingers, person_cache,
post_json_object.copy(), debug,
project_version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem,
sites_unavailable,
system_language), daemon=True)
if not begin_thread(send_thread, 'send_to_followers_thread'):
print('WARN: error while starting ' +
'thread to send to followers.')
return None
return send_thread
def create_inbox(recent_posts_cache: {},
base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, items_per_page: int, header_only: bool,
page_number: int, first_post_id: str) -> {}:
return _create_box_indexed(recent_posts_cache,
base_dir, 'inbox',
nickname, domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number, first_post_id)
def create_bookmarks_timeline(base_dir: str,
nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'tlbookmarks',
nickname, domain,
port, http_prefix, items_per_page, header_only,
True, 0, False, 0, page_number, '')
def create_dm_timeline(recent_posts_cache: {},
base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int,
first_post_id: str) -> {}:
return _create_box_indexed(recent_posts_cache,
base_dir, 'dm', nickname,
domain, port, http_prefix, items_per_page,
header_only, True, 0, False, 0, page_number,
first_post_id)
def create_replies_timeline(recent_posts_cache: {},
base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int,
first_post_id: str) -> {}:
return _create_box_indexed(recent_posts_cache,
base_dir, 'tlreplies',
nickname, domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number, first_post_id)
def create_blogs_timeline(base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'tlblogs', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number, '')
def create_features_timeline(base_dir: str,
nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'tlfeatures', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number, '')
def create_media_timeline(base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'tlmedia', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number, '')
def create_news_timeline(base_dir: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, newswire_votes_threshold: int,
positive_voting: bool, voting_time_mins: int,
page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'outbox', 'news',
domain, port, http_prefix,
items_per_page, header_only, True,
newswire_votes_threshold, positive_voting,
voting_time_mins, page_number, '')
def create_outbox(base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str,
items_per_page: int, header_only: bool, authorized: bool,
page_number: int) -> {}:
return _create_box_indexed({}, base_dir, 'outbox',
nickname, domain, port, http_prefix,
items_per_page, header_only, authorized,
0, False, 0, page_number, '')
def create_moderation(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, items_per_page: int, header_only: bool,
page_number: int) -> {}:
box_dir = create_person_dir(nickname, domain, base_dir, 'inbox')
boxname = 'moderation'
domain = get_full_domain(domain, port)
if not page_number:
page_number = 1
page_str = '?page=' + str(page_number)
box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
box_header = {
'@context': 'https://www.w3.org/ns/activitystreams',
'first': box_url + '?page=true',
'id': box_url,
'last': box_url + '?page=true',
'totalItems': 0,
'type': 'OrderedCollection'
}
box_items = {
'@context': 'https://www.w3.org/ns/activitystreams',
'id': box_url + page_str,
'orderedItems': [
],
'partOf': box_url,
'type': 'OrderedCollectionPage'
}
if is_moderator(base_dir, nickname):
moderation_index_file = data_dir(base_dir) + '/moderation.txt'
if os.path.isfile(moderation_index_file):
with open(moderation_index_file, 'r',
encoding='utf-8') as index_file:
lines = index_file.readlines()
box_header['totalItems'] = len(lines)
if header_only:
return box_header
page_lines = []
if len(lines) > 0:
end_line_number = \
len(lines) - 1 - int(items_per_page * page_number)
end_line_number = max(end_line_number, 0)
start_line_number = \
len(lines) - 1 - int(items_per_page * (page_number - 1))
start_line_number = max(start_line_number, 0)
line_number = start_line_number
while line_number >= end_line_number:
line_no_str = lines[line_number].strip('\n').strip('\r')
page_lines.append(line_no_str)
line_number -= 1
for post_url in page_lines:
post_filename = \
box_dir + '/' + post_url.replace('/', '#') + '.json'
if os.path.isfile(post_filename):
post_json_object = load_json(post_filename)
if post_json_object:
box_items['orderedItems'].append(post_json_object)
if header_only:
return box_header
return box_items
def is_image_media(session, base_dir: str, http_prefix: str,
nickname: str, domain: str,
post_json_object: {},
yt_replace_domain: str,
twitter_replacement_domain: str,
allow_local_network_access: bool,
recent_posts_cache: {}, debug: bool,
system_language: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
bold_reading: bool,
show_vote_posts: bool,
languages_understood: []) -> bool:
"""Returns true if the given post has attached image media
"""
if post_json_object['type'] == 'Announce':
blocked_cache = {}
block_federated = []
post_json_announce = \
download_announce(session, base_dir, http_prefix,
nickname, domain, post_json_object,
__version__,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
blocked_cache, block_federated,
bold_reading,
show_vote_posts,
languages_understood)
if post_json_announce:
post_json_object = post_json_announce
if post_json_object['type'] != 'Create':
return False
if not has_object_dict(post_json_object):
return False
if post_json_object['object'].get('moderationStatus'):
return False
if post_json_object['object']['type'] not in ('Note', 'Page', 'Event',
'ChatMessage', 'Article'):
return False
post_attachments = get_post_attachments(post_json_object)
if not post_attachments:
return False
for attach in post_attachments:
if attach.get('mediaType') and attach.get('url'):
if attach['mediaType'].startswith('image/') or \
attach['mediaType'].startswith('audio/') or \
attach['mediaType'].startswith('video/'):
return True
return False
def _add_post_string_to_timeline(post_str: str, boxname: str,
posts_in_box: [], box_actor: str) -> bool:
""" is this a valid timeline post?
"""
# must be a recognized ActivityPub type
if (string_contains(post_str,
('"Note"', '"EncryptedMessage"', '"ChatMessage"',
'"Event"', '"Article"', '"Patch"', '"Announce"')) or
('"Question"' in post_str and
('"Create"' in post_str or '"Update"' in post_str))):
if boxname == 'dm':
if '#Public' in post_str or \
'/followers' in post_str:
return False
elif boxname == 'tlreplies':
if box_actor not in post_str:
return False
elif boxname in ('tlblogs', 'tlnews', 'tlfeatures'):
if '"Create"' not in post_str:
return False
if '"Article"' not in post_str:
return False
elif boxname == 'tlmedia':
if '"Create"' in post_str:
if ('mediaType' not in post_str or
('image/' not in post_str and
'video/' not in post_str and
'audio/' not in post_str)):
return False
# add the post to the dictionary
posts_in_box.append(post_str)
return True
return False
def _add_post_to_timeline(file_path: str, boxname: str,
posts_in_box: [], box_actor: str) -> bool:
""" Reads a post from file and decides whether it is valid
"""
with open(file_path, 'r', encoding='utf-8') as post_file:
post_str = post_file.read()
if file_path.endswith('.json'):
replies_filename = file_path.replace('.json', '.replies')
if os.path.isfile(replies_filename):
# append a replies identifier, which will later be removed
post_str += '<hasReplies>'
mitm_filename = file_path.replace('.json', '.mitm')
if os.path.isfile(mitm_filename):
# append a mitm identifier, which will later be removed
post_str += '<postmitm>'
return _add_post_string_to_timeline(post_str, boxname, posts_in_box,
box_actor)
return False
def remove_post_interactions(post_json_object: {}, force: bool) -> bool:
""" Don't show likes, replies, bookmarks, DMs or shares (announces) to
unauthorized viewers. This makes the timeline less useful to
marketers and other surveillance-oriented organizations.
Returns False if this is a private post
"""
has_object = False
if has_object_dict(post_json_object):
has_object = True
if has_object:
post_obj = post_json_object['object']
if not force:
# If not authorized and it's a private post
# then just don't show it within timelines
if not is_public_post(post_json_object):
return False
else:
post_obj = post_json_object
# clear the likes
if post_obj.get('likes'):
post_obj['likes'] = {
'items': []
}
# clear the reactions
if post_obj.get('reactions'):
post_obj['reactions'] = {
'items': []
}
# remove other collections
remove_collections = (
'replies', 'shares', 'bookmarks', 'ignores'
)
for remove_name in remove_collections:
if post_obj.get(remove_name):
post_obj[remove_name] = {}
return True
def _passed_newswire_voting(newswire_votes_threshold: int,
base_dir: str, domain: str,
post_filename: str,
positive_voting: bool,
voting_time_mins: int) -> bool:
"""Returns true if the post has passed through newswire voting
"""
# apply votes within this timeline
if newswire_votes_threshold <= 0:
return True
# note that the presence of an arrival file also indicates
# that this post is moderated
arrival_date = \
locate_news_arrival(base_dir, domain, post_filename)
if not arrival_date:
return True
# how long has elapsed since this post arrived?
curr_date = date_utcnow()
time_diff_mins = \
int((curr_date - arrival_date).total_seconds() / 60)
# has the voting time elapsed?
if time_diff_mins < voting_time_mins:
# voting is still happening, so don't add this
# post to the timeline
return False
# if there a votes file for this post?
votes_filename = \
locate_news_votes(base_dir, domain, post_filename)
if not votes_filename:
return True
# load the votes file and count the votes
votes_json = load_json(votes_filename, 0, 2)
if not votes_json:
return True
if not positive_voting:
if votes_on_newswire_item(votes_json) >= \
newswire_votes_threshold:
# Too many veto votes.
# Continue without incrementing
# the posts counter
return False
else:
if votes_on_newswire_item < \
newswire_votes_threshold:
# Not enough votes.
# Continue without incrementing
# the posts counter
return False
return True
def _create_box_items(base_dir: str,
timeline_nickname: str,
original_domain: str,
nickname: str, domain: str,
index_box_name: str,
first_post_id: str,
page_number: int,
items_per_page: int,
newswire_votes_threshold: int,
positive_voting: bool,
voting_time_mins: int,
post_urls_in_box: [],
recent_posts_cache: {},
boxname: str,
posts_in_box: [],
box_actor: str) -> (int, int):
"""Creates the list of posts within a timeline
"""
index_filename = \
acct_dir(base_dir, timeline_nickname, original_domain) + \
'/' + index_box_name + '.index'
total_posts_count = 0
posts_added_to_timeline = 0
if not os.path.isfile(index_filename):
return total_posts_count, posts_added_to_timeline
# format the first post into an hashed url
# Why are url's hashed? Since storage is in the filesystem this avoids
# confusion with directories by not using the / character
if first_post_id:
first_post_id = first_post_id.replace('--', '#')
first_post_id = first_post_id.replace('/', '#')
with open(index_filename, 'r', encoding='utf-8') as index_file:
posts_added_to_timeline = 0
while posts_added_to_timeline < items_per_page:
post_filename = index_file.readline()
if not post_filename:
break
# if a first post is specified then wait until it is found
# before starting to generate the timeline
if first_post_id and total_posts_count == 0:
if first_post_id not in post_filename:
continue
total_posts_count = \
int((page_number - 1) * items_per_page)
# Has this post passed through the newswire voting stage?
if not _passed_newswire_voting(newswire_votes_threshold,
base_dir, domain,
post_filename,
positive_voting,
voting_time_mins):
continue
# Skip through any posts previous to the current page
if not first_post_id:
if total_posts_count < \
int((page_number - 1) * items_per_page):
total_posts_count += 1
continue
# if this is a full path then remove the directories
if '/' in post_filename:
post_filename = post_filename.split('/')[-1]
# filename of the post without any extension or path
# This should also correspond to any index entry in
# the posts cache
post_url = remove_eol(post_filename)
post_url = post_url.replace('.json', '').strip()
# is this a duplicate?
if post_url in post_urls_in_box:
continue
# is the post cached in memory?
if recent_posts_cache.get('index'):
if post_url in recent_posts_cache['index']:
if recent_posts_cache['json'].get(post_url):
url = recent_posts_cache['json'][post_url]
if _add_post_string_to_timeline(url,
boxname,
posts_in_box,
box_actor):
total_posts_count += 1
posts_added_to_timeline += 1
post_urls_in_box.append(post_url)
continue
print('Post not added to timeline')
# read the post from file
full_post_filename = \
locate_post(base_dir, nickname,
original_domain, post_url, False)
if full_post_filename:
# has the post been rejected?
if os.path.isfile(full_post_filename + '.reject'):
continue
if _add_post_to_timeline(full_post_filename, boxname,
posts_in_box, box_actor):
posts_added_to_timeline += 1
total_posts_count += 1
post_urls_in_box.append(post_url)
else:
print('WARN: Unable to add post ' + post_url +
' nickname ' + nickname +
' timeline ' + boxname)
else:
if timeline_nickname != nickname:
# if this is the features timeline
full_post_filename = \
locate_post(base_dir, timeline_nickname,
original_domain, post_url, False)
if full_post_filename:
if _add_post_to_timeline(full_post_filename,
boxname,
posts_in_box, box_actor):
posts_added_to_timeline += 1
total_posts_count += 1
post_urls_in_box.append(post_url)
else:
print('WARN: Unable to add features post ' +
post_url + ' nickname ' + nickname +
' timeline ' + boxname)
else:
print('WARN: features timeline. ' +
'Unable to locate post ' + post_url)
else:
if timeline_nickname == 'news':
print('WARN: Unable to locate news post ' +
post_url + ' nickname ' + nickname)
else:
print('WARN: Unable to locate post ' + post_url +
' nickname ' + nickname)
return total_posts_count, posts_added_to_timeline
def _create_box_indexed(recent_posts_cache: {},
base_dir: str, boxname: str,
nickname: str, domain: str, port: int,
http_prefix: str,
items_per_page: int, header_only: bool,
authorized: bool,
newswire_votes_threshold: int, positive_voting: bool,
voting_time_mins: int, page_number: int,
first_post_id: str) -> {}:
"""Constructs the box feed for a person with the given nickname
"""
if not authorized or not page_number:
page_number = 1
if boxname not in ('inbox', 'dm', 'tlreplies', 'tlmedia',
'tlblogs', 'tlnews', 'tlfeatures', 'outbox',
'tlbookmarks', 'bookmarks'):
print('ERROR: invalid boxname ' + boxname)
return None
unauthorized_premium = False
if not authorized and boxname == 'outbox':
unauthorized_premium = is_premium_account(base_dir, nickname, domain)
# bookmarks and events timelines are like the inbox
# but have their own separate index
index_box_name = boxname
timeline_nickname = nickname
if boxname == "tlbookmarks":
boxname = "bookmarks"
index_box_name = boxname
elif boxname == "tlfeatures":
boxname = "tlblogs"
index_box_name = boxname
timeline_nickname = 'news'
original_domain = domain
domain = get_full_domain(domain, port)
box_actor = local_actor_url(http_prefix, nickname, domain)
page_str = '?page=true'
if page_number:
page_number = max(page_number, 1)
try:
page_str = '?page=' + str(page_number)
except BaseException:
print('EX: _create_box_indexed ' +
'unable to convert page number to string')
box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
box_header = {
'@context': 'https://www.w3.org/ns/activitystreams',
'first': box_url + '?page=true',
'id': box_url,
'last': box_url + '?page=true',
'totalItems': 0,
'type': 'OrderedCollection'
}
box_items = {
'@context': 'https://www.w3.org/ns/activitystreams',
'id': box_url + page_str,
'orderedItems': [
],
'partOf': box_url,
'type': 'OrderedCollectionPage'
}
posts_in_box = []
post_urls_in_box = []
if not unauthorized_premium:
total_posts_count, posts_added_to_timeline = \
_create_box_items(base_dir,
timeline_nickname,
original_domain,
nickname, domain,
index_box_name,
first_post_id,
page_number,
items_per_page,
newswire_votes_threshold,
positive_voting,
voting_time_mins,
post_urls_in_box,
recent_posts_cache,
boxname,
posts_in_box,
box_actor)
else:
# unauthorized requests for premium posts should return an empty set
total_posts_count = posts_added_to_timeline = 0
if first_post_id and \
posts_added_to_timeline == 0 and \
not unauthorized_premium:
# no first post was found within the index, so just use the page number
first_post_id = ''
total_posts_count, posts_added_to_timeline = \
_create_box_items(base_dir,
timeline_nickname,
original_domain,
nickname, domain,
index_box_name,
None,
page_number,
items_per_page,
newswire_votes_threshold,
positive_voting,
voting_time_mins,
post_urls_in_box,
recent_posts_cache,
boxname,
posts_in_box,
box_actor)
if total_posts_count < 3:
print('Posts added to json timeline ' + boxname + ': ' +
str(posts_added_to_timeline))
# Generate first and last entries within header
if total_posts_count > 0:
last_page = int(total_posts_count / items_per_page)
last_page = max(last_page, 1)
box_header['last'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + str(last_page)
if header_only:
box_header['totalItems'] = len(posts_in_box)
prev_page_str = 'true'
if page_number > 1:
prev_page_str = str(page_number - 1)
box_header['prev'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + prev_page_str
next_page_str = str(page_number + 1)
box_header['next'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + next_page_str
return box_header
for post_str in posts_in_box:
# Check if the post has replies
has_replies = False
if post_str.endswith('<hasReplies>'):
has_replies = True
# remove the replies identifier
post_str = post_str.replace('<hasReplies>', '')
# Check if the post was delivered via a third party
mitm = False
if post_str.endswith('<postmitm>'):
mitm = True
# remove the mitm identifier
post_str = post_str.replace('<postmitm>', '')
pst = None
try:
pst = json.loads(post_str)
except BaseException:
print('EX: _create_box_indexed unable to load json ' + post_str)
continue
# Does this post have replies?
# This will be used to indicate that replies exist within the html
# created by individual_post_as_html
pst['hasReplies'] = has_replies
# was the post delivered via a third party?
pst['mitm'] = mitm
if not authorized:
if not remove_post_interactions(pst, False):
continue
box_items['orderedItems'].append(pst)
return box_items
def expire_cache(base_dir: str, person_cache: {},
http_prefix: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box: int,
max_cache_age_days: int):
"""Thread used to expire actors from the cache and archive old posts
"""
while True:
# once per day
time.sleep(60 * 60 * 24)
expire_person_cache(person_cache)
archive_posts(base_dir, http_prefix, archive_dir, recent_posts_cache,
max_posts_in_box, max_cache_age_days)
def _expire_announce_cache_for_person(base_dir: str,
nickname: str, domain: str,
max_age_days: int) -> int:
"""Expires entries within the announces cache
"""
cache_dir = base_dir + '/cache/announce/' + nickname
if not os.path.isdir(cache_dir):
print('No cached announces for ' + nickname + '@' + domain)
return 0
expired_post_count = 0
posts_in_cache = os.scandir(cache_dir)
for cache_filename in posts_in_cache:
cache_filename = cache_filename.name
# Time of file creation
full_filename = os.path.join(cache_dir, cache_filename)
if not os.path.isfile(full_filename):
continue
last_modified = file_last_modified(full_filename)
# get time difference
if not valid_post_date(last_modified, max_age_days, False):
try:
os.remove(full_filename)
except OSError:
print('EX: unable to delete from announce cache ' +
full_filename)
expired_post_count += 1
return expired_post_count
def _expire_posts_cache_for_person(base_dir: str,
nickname: str, domain: str,
max_age_days: int) -> int:
"""Expires entries within the posts cache
"""
cache_dir = acct_dir(base_dir, nickname, domain) + '/postcache'
if not os.path.isdir(cache_dir):
print('No cached posts for ' + nickname + '@' + domain)
return 0
expired_post_count = 0
posts_in_cache = os.scandir(cache_dir)
for cache_filename in posts_in_cache:
cache_filename = cache_filename.name
# Time of file creation
full_filename = os.path.join(cache_dir, cache_filename)
if not os.path.isfile(full_filename):
continue
last_modified = file_last_modified(full_filename)
# get time difference
if not valid_post_date(last_modified, max_age_days, False):
try:
os.remove(full_filename)
except OSError:
print('EX: unable to delete from post cache ' +
full_filename)
expired_post_count += 1
return expired_post_count
def archive_posts(base_dir: str, http_prefix: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box: int,
max_cache_age_days: int) -> None:
"""Archives posts for all accounts
"""
if max_posts_in_box == 0:
return
if archive_dir:
if not os.path.isdir(archive_dir):
os.mkdir(archive_dir)
if archive_dir:
if not os.path.isdir(archive_dir + '/accounts'):
os.mkdir(archive_dir + '/accounts')
dir_str = data_dir(base_dir)
for _, dirs, _ in os.walk(dir_str):
for handle in dirs:
if '@' in handle:
nickname = handle.split('@')[0]
domain = handle.split('@')[1]
archive_subdir = None
if archive_dir:
archive_handle_dir = acct_handle_dir(archive_dir, handle)
if not os.path.isdir(archive_handle_dir):
os.mkdir(archive_handle_dir)
if not os.path.isdir(archive_handle_dir + '/inbox'):
os.mkdir(archive_handle_dir + '/inbox')
if not os.path.isdir(archive_handle_dir + '/outbox'):
os.mkdir(archive_handle_dir + '/outbox')
archive_subdir = archive_handle_dir + '/inbox'
archive_posts_for_person(http_prefix,
nickname, domain, base_dir,
'inbox', archive_subdir,
recent_posts_cache, max_posts_in_box)
expired_announces = \
_expire_announce_cache_for_person(base_dir,
nickname, domain,
max_cache_age_days)
print('Expired ' + str(expired_announces) +
' cached announces for ' + nickname + '@' + domain)
expired_posts = \
_expire_posts_cache_for_person(base_dir,
nickname, domain,
max_cache_age_days)
print('Expired ' + str(expired_posts) +
' cached posts for ' + nickname + '@' + domain)
if archive_dir:
archive_subdir = archive_dir + '/accounts/' + \
handle + '/outbox'
archive_posts_for_person(http_prefix,
nickname, domain, base_dir,
'outbox', archive_subdir,
recent_posts_cache, max_posts_in_box)
break
def _expire_posts_for_person(http_prefix: str, nickname: str, domain: str,
base_dir: str, recent_posts_cache: {},
max_age_days: int, debug: bool,
keep_dms: bool) -> int:
"""Removes posts older than some number of days
"""
expired_post_count = 0
if max_age_days <= 0:
return expired_post_count
boxname = 'outbox'
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
posts_in_box = os.scandir(box_dir)
for post_filename in posts_in_box:
post_filename = post_filename.name
if not post_filename.endswith('.json'):
continue
# Time of file creation
full_filename = os.path.join(box_dir, post_filename)
if not os.path.isfile(full_filename):
continue
content = ''
try:
with open(full_filename, 'r', encoding='utf-8') as fp_content:
content = fp_content.read()
except OSError:
print('EX: expire_posts_for_person unable to open content ' +
full_filename)
if '"published":' not in content:
continue
published_str = content.split('"published":')[1]
if '"' not in published_str:
continue
published_str = published_str.split('"')[1]
if not published_str.endswith('Z'):
continue
# get time difference
if not valid_post_date(published_str, max_age_days, debug):
if keep_dms:
post_json_object = load_json(full_filename)
if not post_json_object:
continue
if is_dm(post_json_object):
continue
delete_post(base_dir, http_prefix, nickname, domain,
full_filename, debug, recent_posts_cache, True)
expired_post_count += 1
return expired_post_count
def get_post_expiry_keep_dms(base_dir: str, nickname: str, domain: str) -> int:
"""Returns true if dms should expire
"""
keep_dms = True
handle = nickname + '@' + domain
expire_dms_filename = \
acct_handle_dir(base_dir, handle) + '/.expire_posts_dms'
if os.path.isfile(expire_dms_filename):
keep_dms = False
return keep_dms
def set_post_expiry_keep_dms(base_dir: str, nickname: str, domain: str,
keep_dms: bool) -> None:
"""Sets whether to keep DMs during post expiry for an account
"""
handle = nickname + '@' + domain
expire_dms_filename = \
acct_handle_dir(base_dir, handle) + '/.expire_posts_dms'
if keep_dms:
if os.path.isfile(expire_dms_filename):
try:
os.remove(expire_dms_filename)
except OSError:
print('EX: unable to write set_post_expiry_keep_dms False ' +
expire_dms_filename)
return
try:
with open(expire_dms_filename, 'w+', encoding='utf-8') as fp_expire:
fp_expire.write('\n')
except OSError:
print('EX: unable to write set_post_expiry_keep_dms True ' +
expire_dms_filename)
def expire_posts(base_dir: str, http_prefix: str,
recent_posts_cache: {}, debug: bool) -> int:
"""Expires posts for instance accounts
"""
expired_post_count = 0
dir_str = data_dir(base_dir)
for _, dirs, _ in os.walk(dir_str):
for handle in dirs:
if '@' not in handle:
continue
nickname = handle.split('@')[0]
domain = handle.split('@')[1]
expire_posts_filename = \
acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
if not os.path.isfile(expire_posts_filename):
continue
keep_dms = get_post_expiry_keep_dms(base_dir, nickname, domain)
expire_days_str = None
try:
with open(expire_posts_filename, 'r',
encoding='utf-8') as fp_expire:
expire_days_str = fp_expire.read()
except OSError:
print('EX: expire_posts failed to read days file ' +
expire_posts_filename)
continue
if not expire_days_str:
continue
if not expire_days_str.isdigit():
continue
max_age_days = int(expire_days_str)
if max_age_days <= 0:
continue
expired_post_count += \
_expire_posts_for_person(http_prefix,
nickname, domain, base_dir,
recent_posts_cache,
max_age_days, debug,
keep_dms)
break
return expired_post_count
def get_post_expiry_days(base_dir: str, nickname: str, domain: str) -> int:
"""Returns the post expiry period for the given account
"""
handle = nickname + '@' + domain
expire_posts_filename = \
acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
if not os.path.isfile(expire_posts_filename):
return 0
days_str = None
try:
with open(expire_posts_filename, 'r', encoding='utf-8') as fp_expire:
days_str = fp_expire.read()
except OSError:
print('EX: unable to write post expire days ' +
expire_posts_filename)
if not days_str:
return 0
if not days_str.isdigit():
return 0
return int(days_str)
def set_post_expiry_days(base_dir: str, nickname: str, domain: str,
max_age_days: int) -> None:
"""Sets the number of days after which posts from an account will expire
"""
handle = nickname + '@' + domain
expire_posts_filename = \
acct_handle_dir(base_dir, handle) + '/.expire_posts_days'
try:
with open(expire_posts_filename, 'w+', encoding='utf-8') as fp_expire:
fp_expire.write(str(max_age_days))
except OSError:
print('EX: unable to write post expire days ' +
expire_posts_filename)
def archive_posts_for_person(http_prefix: str, nickname: str, domain: str,
base_dir: str,
boxname: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box=32000) -> None:
"""Retain a maximum number of posts within the given box
Move any others to an archive directory
"""
if boxname not in ('inbox', 'outbox'):
return
if archive_dir:
if not os.path.isdir(archive_dir):
os.mkdir(archive_dir)
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
posts_in_box = os.scandir(box_dir)
no_of_posts = 0
for _ in posts_in_box:
no_of_posts += 1
if no_of_posts <= max_posts_in_box:
print('Checked ' + str(no_of_posts) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
return
# remove entries from the index
handle = nickname + '@' + domain
index_filename = \
acct_handle_dir(base_dir, handle) + '/' + boxname + '.index'
if os.path.isfile(index_filename):
index_ctr = 0
# get the existing index entries as a string
new_index = ''
try:
with open(index_filename, 'r', encoding='utf-8') as index_file:
for post_id in index_file:
new_index += post_id
index_ctr += 1
if index_ctr >= max_posts_in_box:
break
except OSError as ex:
print('EX: archive_posts_for_person unable to read ' +
index_filename + ' ' + str(ex))
# save the new index file
if len(new_index) > 0:
try:
with open(index_filename, 'w+',
encoding='utf-8') as index_file:
index_file.write(new_index)
except OSError:
print('EX: archive_posts_for_person unable to write ' +
index_filename)
posts_in_box_dict = {}
posts_ctr = 0
posts_in_box = os.scandir(box_dir)
for post_filename in posts_in_box:
post_filename = post_filename.name
if not post_filename.endswith('.json'):
continue
# Time of file creation
full_filename = os.path.join(box_dir, post_filename)
if os.path.isfile(full_filename):
content = ''
try:
with open(full_filename, 'r', encoding='utf-8') as fp_content:
content = fp_content.read()
except OSError:
print('EX: unable to open content ' + full_filename)
if '"published":' in content:
published_str = content.split('"published":')[1]
if '"' in published_str:
published_str = published_str.split('"')[1]
if published_str.endswith('Z'):
posts_in_box_dict[published_str] = post_filename
posts_ctr += 1
no_of_posts = posts_ctr
if no_of_posts <= max_posts_in_box:
print('Checked ' + str(no_of_posts) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
return
# sort the list in ascending order of date
posts_in_box_sorted = \
OrderedDict(sorted(posts_in_box_dict.items(), reverse=False))
# directory containing cached html posts
post_cache_dir = box_dir.replace('/' + boxname, '/postcache')
remove_ctr = 0
for published_str, post_filename in posts_in_box_sorted.items():
file_path = os.path.join(box_dir, post_filename)
if not os.path.isfile(file_path):
continue
if archive_dir:
archive_path = os.path.join(archive_dir, post_filename)
os.rename(file_path, archive_path)
extensions = ('replies', 'votes', 'arrived', 'muted')
for ext in extensions:
ext_path = file_path.replace('.json', '.' + ext)
if os.path.isfile(ext_path):
os.rename(ext_path,
archive_path.replace('.json', '.' + ext))
else:
ext_path = file_path.replace('.json',
'.json.' + ext)
if os.path.isfile(ext_path):
os.rename(ext_path,
archive_path.replace('.json',
'.json.' + ext))
else:
delete_post(base_dir, http_prefix, nickname, domain,
file_path, False, recent_posts_cache, False)
# remove cached html posts
post_cache_filename = \
os.path.join(post_cache_dir, post_filename)
post_cache_filename = post_cache_filename.replace('.json', '.html')
if os.path.isfile(post_cache_filename):
try:
os.remove(post_cache_filename)
except OSError:
print('EX: archive_posts_for_person unable to delete ' +
post_cache_filename)
no_of_posts -= 1
remove_ctr += 1
if no_of_posts <= max_posts_in_box:
break
if archive_dir:
print('Archived ' + str(remove_ctr) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
else:
print('Removed ' + str(remove_ctr) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
print(nickname + '@' + domain + ' has ' + str(no_of_posts) +
' in ' + boxname)
def get_public_posts_of_person(base_dir: str, nickname: str, domain: str,
raw: bool, simple: bool, proxy_type: str,
port: int, http_prefix: str,
debug: bool, project_version: str,
system_language: str,
signing_priv_key_pem: str,
origin_domain: str) -> None:
""" This is really just for test purposes
"""
if debug:
if signing_priv_key_pem:
print('Signing key available')
else:
print('Signing key missing')
if debug:
print('Starting new session for getting public posts')
session = create_session(proxy_type)
if not session:
if debug:
print('Session was not created')
return
person_cache = {}
cached_webfingers = {}
federation_list = []
group_account = False
if nickname.startswith('!'):
nickname = nickname[1:]
group_account = True
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
origin_domain, project_version, debug, group_account,
signing_priv_key_pem)
if not wf_request:
if debug:
print('No webfinger result was returned for ' + handle)
sys.exit()
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
sys.exit()
if debug:
print('\nGetting the outbox for ' + handle)
(person_url, _, _, person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
62524, system_language)
if debug:
print('Actor url: ' + str(person_id))
if not person_id:
return
max_mentions = 10
max_emoji = 10
max_attachments = 5
_get_posts(session, person_url, 30, max_mentions, max_emoji,
max_attachments, federation_list, raw, simple, debug,
project_version, http_prefix, origin_domain, system_language,
signing_priv_key_pem)
def get_public_post_domains(session, base_dir: str, nickname: str, domain: str,
origin_domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, domain_list: [],
system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a list of domains referenced within public posts
"""
if not session:
session = create_session(proxy_type)
if not session:
return domain_list
person_cache = {}
cached_webfingers = {}
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return domain_list
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return domain_list
(person_url, _, _, _, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
92522, system_language)
post_domains = \
get_post_domains(session, person_url, 64, debug,
project_version, http_prefix, domain,
word_frequency, domain_list, system_language,
signing_priv_key_pem)
post_domains.sort()
return post_domains
def download_follow_collection(signing_priv_key_pem: str,
follow_type: str,
session, http_prefix: str,
actor: str, page_number: int,
no_of_pages: int, debug: bool) -> []:
"""Returns a list of following/followers for the given actor
by downloading the json for their following/followers collection
"""
prof = 'https://www.w3.org/ns/activitystreams'
if '/channel/' not in actor or '/accounts/' not in actor:
accept_str = \
'application/activity+json; ' + \
'profile="' + prof + '"'
session_headers = {
'Accept': accept_str
}
else:
accept_str = \
'application/ld+json; ' + \
'profile="' + prof + '"'
session_headers = {
'Accept': accept_str
}
result = []
for page_ctr in range(no_of_pages):
url = \
actor + '/' + follow_type + '?page=' + str(page_number + page_ctr)
followers_json = \
get_json(signing_priv_key_pem, session, url, session_headers, None,
debug, __version__, http_prefix, None)
if get_json_valid(followers_json):
if followers_json.get('orderedItems'):
for follower_actor in followers_json['orderedItems']:
if follower_actor not in result:
result.append(follower_actor)
elif followers_json.get('items'):
for follower_actor in followers_json['items']:
if follower_actor not in result:
result.append(follower_actor)
else:
break
else:
break
return result
def get_public_post_info(session, base_dir: str, nickname: str, domain: str,
origin_domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a dict of domains referenced within public posts
"""
if not session:
session = create_session(proxy_type)
if not session:
return {}
person_cache = {}
cached_webfingers = {}
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return {}
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return {}
(person_url, _, _, _, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
13863, system_language)
max_posts = 64
post_domains = \
get_post_domains(session, person_url, max_posts, debug,
project_version, http_prefix, domain,
word_frequency, [], system_language,
signing_priv_key_pem)
post_domains.sort()
domains_info = {}
for pdomain in post_domains:
if not domains_info.get(pdomain):
domains_info[pdomain] = []
blocked_posts = \
_get_posts_for_blocked_domains(base_dir, session,
person_url, max_posts,
debug,
project_version, http_prefix,
domain, signing_priv_key_pem)
for blocked_domain, post_url_list in blocked_posts.items():
domains_info[blocked_domain] += post_url_list
return domains_info
def get_public_post_domains_blocked(session, base_dir: str,
nickname: str, domain: str,
proxy_type: str, port: int,
http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, domain_list: [],
system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a list of domains referenced within public posts which
are globally blocked on this instance
"""
origin_domain = domain
post_domains = \
get_public_post_domains(session, base_dir, nickname, domain,
origin_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, domain_list, system_language,
signing_priv_key_pem)
if not post_domains:
return []
blocking_filename = data_dir(base_dir) + '/blocking.txt'
if not os.path.isfile(blocking_filename):
return []
# read the blocked domains as a single string
blocked_str = ''
with open(blocking_filename, 'r', encoding='utf-8') as fp_block:
blocked_str = fp_block.read()
blocked_domains = []
for domain_name in post_domains:
if '@' not in domain_name:
continue
# get the domain after the @
domain_name = domain_name.split('@')[1].strip()
if is_evil(domain_name):
blocked_domains.append(domain_name)
continue
if domain_name in blocked_str:
blocked_domains.append(domain_name)
return blocked_domains
def _get_non_mutuals_of_person(base_dir: str,
nickname: str, domain: str) -> []:
"""Returns the followers who are not mutuals of a person
i.e. accounts which follow you but you don't follow them
"""
followers = \
get_followers_list(base_dir, nickname, domain, 'followers.txt')
following = \
get_followers_list(base_dir, nickname, domain, 'following.txt')
non_mutuals = []
for handle in followers:
if handle not in following:
non_mutuals.append(handle)
return non_mutuals
def check_domains(session, base_dir: str,
nickname: str, domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
max_blocked_domains: int, single_check: bool,
system_language: str,
signing_priv_key_pem: str) -> None:
"""Checks follower accounts for references to globally blocked domains
"""
word_frequency = {}
non_mutuals = _get_non_mutuals_of_person(base_dir, nickname, domain)
if not non_mutuals:
print('No non-mutual followers were found')
return
follower_warning_filename = data_dir(base_dir) + '/followerWarnings.txt'
update_follower_warnings = False
follower_warning_str = ''
if os.path.isfile(follower_warning_filename):
with open(follower_warning_filename, 'r',
encoding='utf-8') as fp_warn:
follower_warning_str = fp_warn.read()
if single_check:
# checks a single random non-mutual
index = random.randrange(0, len(non_mutuals))
handle = non_mutuals[index]
if '@' in handle:
non_mutual_nickname = handle.split('@')[0]
non_mutual_domain = handle.split('@')[1].strip()
blocked_domains = \
get_public_post_domains_blocked(session, base_dir,
non_mutual_nickname,
non_mutual_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, [],
system_language,
signing_priv_key_pem)
if blocked_domains:
if len(blocked_domains) > max_blocked_domains:
follower_warning_str += handle + '\n'
update_follower_warnings = True
else:
# checks all non-mutuals
for handle in non_mutuals:
if '@' not in handle:
continue
if handle in follower_warning_str:
continue
non_mutual_nickname = handle.split('@')[0]
non_mutual_domain = handle.split('@')[1].strip()
blocked_domains = \
get_public_post_domains_blocked(session, base_dir,
non_mutual_nickname,
non_mutual_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, [],
system_language,
signing_priv_key_pem)
if blocked_domains:
print(handle)
for bdomain in blocked_domains:
print(' ' + bdomain)
if len(blocked_domains) > max_blocked_domains:
follower_warning_str += handle + '\n'
update_follower_warnings = True
if update_follower_warnings and follower_warning_str:
try:
with open(follower_warning_filename, 'w+',
encoding='utf-8') as fp_warn:
fp_warn.write(follower_warning_str)
except OSError:
print('EX: check_domains unable to write ' +
follower_warning_filename)
if not single_check:
print(follower_warning_str)
def populate_replies_json(base_dir: str, nickname: str, domain: str,
post_replies_filename: str, authorized: bool,
replies_json: {}) -> None:
pub_str = 'https://www.w3.org/ns/activitystreams#Public'
# populate the items list with replies
replies_boxes = ('outbox', 'inbox')
with open(post_replies_filename, 'r', encoding='utf-8') as replies_file:
for message_id in replies_file:
reply_found = False
# examine inbox and outbox
for boxname in replies_boxes:
message_id2 = remove_eol(message_id)
search_filename = \
acct_dir(base_dir, nickname, domain) + '/' + \
boxname + '/' + \
message_id2.replace('/', '#') + '.json'
if os.path.isfile(search_filename):
if authorized or \
text_in_file(pub_str, search_filename):
post_json_object = load_json(search_filename)
if post_json_object:
if post_json_object['object'].get('cc'):
pjo = post_json_object
if (authorized or
(pub_str in pjo['object']['to'] or
pub_str in pjo['object']['cc'])):
replies_json['orderedItems'].append(pjo)
reply_found = True
else:
if authorized or \
pub_str in post_json_object['object']['to']:
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
reply_found = True
break
# if not in either inbox or outbox then examine the shared inbox
if not reply_found:
message_id2 = remove_eol(message_id)
search_filename = \
data_dir(base_dir) + '/inbox@' + \
domain + '/inbox/' + \
message_id2.replace('/', '#') + '.json'
if os.path.isfile(search_filename):
if authorized or \
text_in_file(pub_str, search_filename):
# get the json of the reply and append it to
# the collection
post_json_object = load_json(search_filename)
if post_json_object:
if post_json_object['object'].get('cc'):
pjo = post_json_object
if (authorized or
(pub_str in pjo['object']['to'] or
pub_str in pjo['object']['cc'])):
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
else:
if authorized or \
pub_str in post_json_object['object']['to']:
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
def _reject_announce(announce_filename: str,
base_dir: str, nickname: str, domain: str,
announce_post_id: str, recent_posts_cache: {}):
"""Marks an announce as rejected
"""
reject_post_id(base_dir, nickname, domain, announce_post_id,
recent_posts_cache)
# reject the post referenced by the announce activity object
if not os.path.isfile(announce_filename + '.reject'):
try:
with open(announce_filename + '.reject', 'w+',
encoding='utf-8') as reject_announce_file:
reject_announce_file.write('\n')
except OSError:
print('EX: _reject_announce unable to write ' +
announce_filename + '.reject')
def download_announce(session, base_dir: str, http_prefix: str,
nickname: str, domain: str,
post_json_object: {}, project_version: str,
yt_replace_domain: str,
twitter_replacement_domain: str,
allow_local_network_access: bool,
recent_posts_cache: {}, debug: bool,
system_language: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
blocked_cache: {}, block_federated: [],
bold_reading: bool,
show_vote_posts: bool,
languages_understood: []) -> {}:
"""Download the post referenced by an announce
"""
if not post_json_object.get('object'):
return None
if not isinstance(post_json_object['object'], str):
return None
# ignore self-boosts
actor_url = get_actor_from_post(post_json_object)
if actor_url in post_json_object['object']:
return None
# get the announced post
announce_cache_dir = base_dir + '/cache/announce/' + nickname
if not os.path.isdir(announce_cache_dir):
os.mkdir(announce_cache_dir)
post_id = None
if post_json_object.get('id'):
post_id = remove_id_ending(post_json_object['id'])
announce_filename = \
announce_cache_dir + '/' + \
post_json_object['object'].replace('/', '#') + '.json'
if os.path.isfile(announce_filename + '.reject'):
return None
if os.path.isfile(announce_filename):
if debug:
print('Reading cached Announce content for ' +
post_json_object['object'])
post_json_object = load_json(announce_filename)
if post_json_object:
return post_json_object
else:
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/channel/' in actor_url or \
'/accounts/' in actor_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
actor_nickname = get_nickname_from_actor(actor_url)
if not actor_nickname:
print('WARN: download_announce no actor_nickname')
return None
actor_domain, actor_port = \
get_domain_from_actor(actor_url)
if not actor_domain:
print('Announce actor does not contain a ' +
'valid domain or port number: ' +
actor_url)
return None
if is_blocked(base_dir, nickname, domain,
actor_nickname, actor_domain,
None, block_federated):
print('Announce download blocked actor: ' +
actor_nickname + '@' + actor_domain)
return None
object_nickname = get_nickname_from_actor(post_json_object['object'])
object_domain, _ = \
get_domain_from_actor(post_json_object['object'])
if not object_domain:
print('Announce object does not contain a ' +
'valid domain or port number: ' +
str(post_json_object['object']))
return None
if is_blocked(base_dir, nickname, domain, object_nickname,
object_domain, None, block_federated):
if object_nickname and object_domain:
print('Announce download blocked handle: ' +
object_nickname + '@' + object_domain)
else:
print('Announce download blocked handle 2: ' +
str(post_json_object['object']))
return None
if debug:
print('Downloading Announce content for ' +
post_json_object['object'])
announced_json = \
get_json(signing_priv_key_pem, session,
post_json_object['object'],
as_header, None, debug, project_version,
http_prefix, domain)
if not get_json_valid(announced_json):
return None
if not isinstance(announced_json, dict):
print('WARN: announced post json is not a dict - ' +
post_json_object['object'] + ' ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json.get('error'):
print('WARN: ' +
'Attempt to download announce returned an error ' +
post_json_object['object'] + ' ' +
str(announced_json))
return None
if not announced_json.get('id'):
print('WARN: announced post does not have an id ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
announced_actor = announced_json['id']
if announced_json.get('attributedTo'):
announced_actor = get_attributed_to(announced_json['attributedTo'])
if not announced_json.get('type'):
print('WARN: announced post does not have a type ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json['type'] == 'Video':
converted_json = \
convert_video_to_note(base_dir, nickname, domain,
system_language,
announced_json, blocked_cache,
block_federated,
languages_understood)
if converted_json:
announced_json = converted_json
if not contains_statuses(announced_json['id']):
print('WARN: announced post id does not contain /statuses/ ' +
'or /objects/ or /p/ ' + str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not has_users_path(announced_actor):
print('WARN: announced post id does not contain /users/ ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json['type'] not in ('Note', 'Event', 'Page',
'Question', 'Article'):
print('WARN: announced post is not Note/Page/Article/Question ' +
str(announced_json))
# You can only announce Note, Page, Article or Question types
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json['type'] == 'Question':
if not announced_json.get('endTime') or \
not announced_json.get('oneOf'):
# announced Question should have an end time
# and options list
print('WARN: announced Question should have endTime ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not isinstance(announced_json['oneOf'], list):
print('WARN: announced Question oneOf should be a list ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if is_question_filtered(base_dir, nickname, domain,
system_language, announced_json):
if announced_json.get('id'):
print('REJECT: announced question was filtered ' +
str(announced_json['id']))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if 'content' not in announced_json:
print('WARN: announced post does not have content ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not announced_json.get('published'):
print('WARN: announced post does not have published ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if '.' in announced_json['published'] and \
'Z' in announced_json['published']:
announced_json['published'] = \
announced_json['published'].split('.')[0] + 'Z'
if not valid_post_date(announced_json['published'], 90, debug):
print('WARN: announced post is not recently published ' +
str(announced_json['published']))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not understood_post_language(base_dir, nickname,
announced_json, system_language,
http_prefix, domain_full,
person_cache):
return None
# Check the content of the announce
convert_post_content_to_html(announced_json)
content_str = announced_json['content']
using_content_map = False
if 'contentMap' in announced_json:
if announced_json['contentMap'].get(system_language):
content_str = announced_json['contentMap'][system_language]
using_content_map = True
if dangerous_markup(content_str, allow_local_network_access, []):
print('WARN: announced post contains dangerous markup ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
summary_str = \
get_summary_from_post(announced_json, system_language, [])
media_descriptions = \
get_media_descriptions_from_post(announced_json)
content_all = content_str
if summary_str:
content_all = \
summary_str + ' ' + content_str + ' ' + media_descriptions
if is_filtered(base_dir, nickname, domain, content_all,
system_language):
if announced_json.get('id'):
print('REJECT: announced post has been filtered ' +
str(announced_json['id']))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if reject_twitter_summary(base_dir, nickname, domain,
summary_str):
print('WARN: announced post has twitter summary ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if contains_private_key(content_str):
print("WARN: announced post contains someone's private key " +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if invalid_ciphertext(content_str):
print('WARN: announced post contains invalid ciphertext ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
# remove any long words
content_str = remove_long_words(content_str, 40, [])
# Prevent the same word from being repeated many times
content_str = limit_repeated_words(content_str, 6)
# remove text formatting, such as bold/italics
content_str = remove_text_formatting(content_str, bold_reading)
# set the content after santitization
if using_content_map:
announced_json['contentMap'][system_language] = content_str
announced_json['content'] = content_str
# wrap in create to be consistent with other posts
announced_json = \
outbox_message_create_wrap(http_prefix,
actor_nickname, actor_domain,
actor_port, announced_json)
if announced_json['type'] != 'Create':
print('WARN: announced post could not be wrapped in Create ' +
str(announced_json))
# Create wrap failed
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
# if poll/vote/question is not to be shown
if not show_vote_posts:
if is_question(announced_json):
return None
# set the id to the original status
announced_json['id'] = post_json_object['object']
announced_json['object']['id'] = post_json_object['object']
# check that the repeat isn't for a blocked account
attributed_nickname = \
get_nickname_from_actor(announced_json['object']['id'])
attributed_domain, attributed_port = \
get_domain_from_actor(announced_json['object']['id'])
if attributed_nickname and attributed_domain:
attributed_domain = \
get_full_domain(attributed_domain, attributed_port)
if is_blocked(base_dir, nickname, domain,
attributed_nickname, attributed_domain,
None, block_federated):
print('WARN: announced post handle is blocked ' +
str(attributed_nickname) + '@' + attributed_domain)
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
post_json_object = announced_json
replace_you_tube(post_json_object, yt_replace_domain, system_language)
replace_twitter(post_json_object, twitter_replacement_domain,
system_language)
if save_json(post_json_object, announce_filename):
return post_json_object
return None
def is_muted_conv(base_dir: str, nickname: str, domain: str, post_id: str,
conversation_id: str) -> bool:
"""Returns true if the given post is muted
"""
if conversation_id:
conv_muted_filename = \
acct_dir(base_dir, nickname, domain) + '/conversation/' + \
conversation_id.replace('/', '#') + '.muted'
if os.path.isfile(conv_muted_filename):
return True
post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename:
return False
if os.path.isfile(post_filename + '.muted'):
return True
return False
def send_block_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, blocked_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
system_language: str) -> {}:
"""Creates a block via c2s
"""
if not session:
print('WARN: No session for send_block_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
to_url = 'https://www.w3.org/ns/activitystreams#Public'
cc_url = block_actor + '/followers'
new_block_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Block',
'actor': block_actor,
'object': blocked_url,
'to': [to_url],
'cc': [cc_url]
}
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: block webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: block Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652,
system_language)
if not inbox_url:
if debug:
print('DEBUG: block no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: block no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_block_json, [], inbox_url,
headers, 30, True)
if not post_result:
print('WARN: block unable to post')
if debug:
print('DEBUG: c2s POST block success')
return new_block_json
def send_mute_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, muted_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
system_language: str) -> {}:
"""Creates a mute via c2s
"""
if not session:
print('WARN: No session for send_mute_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
handle = replace_users_with_at(actor)
new_mute_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Ignore',
'actor': actor,
'to': [actor],
'object': muted_url
}
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: mute webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: mute Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652,
system_language)
if not inbox_url:
if debug:
print('DEBUG: mute no ' + post_to_box + ' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: mute no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_mute_json, [], inbox_url,
headers, 3, True)
if post_result is None:
print('WARN: mute unable to post')
if debug:
print('DEBUG: c2s POST mute success')
return new_mute_json
def send_undo_mute_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, muted_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
system_language: str) -> {}:
"""Undoes a mute via c2s
"""
if not session:
print('WARN: No session for send_undo_mute_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
handle = replace_users_with_at(actor)
undo_mute_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Undo',
'actor': actor,
'to': [actor],
'object': {
'type': 'Ignore',
'actor': actor,
'to': [actor],
'object': muted_url
}
}
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: undo mute webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: undo mute Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652,
system_language)
if not inbox_url:
if debug:
print('DEBUG: undo mute no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: undo mute no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, undo_mute_json, [], inbox_url,
headers, 3, True)
if post_result is None:
print('WARN: undo mute unable to post')
if debug:
print('DEBUG: c2s POST undo mute success')
return undo_mute_json
def send_undo_block_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, blocked_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
system_language: str) -> {}:
"""Creates a block via c2s
"""
if not session:
print('WARN: No session for send_block_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
to_url = 'https://www.w3.org/ns/activitystreams#Public'
cc_url = block_actor + '/followers'
new_block_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Undo',
'actor': block_actor,
'object': {
'type': 'Block',
'actor': block_actor,
'object': blocked_url,
'to': [to_url],
'cc': [cc_url]
}
}
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: unblock webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: unblock webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 53892,
system_language)
if not inbox_url:
if debug:
print('DEBUG: unblock no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: unblock no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_block_json, [], inbox_url,
headers, 30, True)
if not post_result:
print('WARN: unblock unable to post')
if debug:
print('DEBUG: c2s POST unblock success')
return new_block_json
def post_is_muted(base_dir: str, nickname: str, domain: str,
post_json_object: {}, message_id: str) -> bool:
""" Returns true if the given post is muted
"""
is_muted = None
if 'muted' in post_json_object:
is_muted = post_json_object['muted']
if is_muted is True or is_muted is False:
return is_muted
is_muted = False
post_dir = acct_dir(base_dir, nickname, domain)
mute_filename = \
post_dir + '/inbox/' + message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
else:
mute_filename = \
post_dir + '/outbox/' + \
message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
else:
mute_filename = \
data_dir(base_dir) + '/cache/announce/' + nickname + \
'/' + message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
return is_muted
def c2s_box_json(session, nickname: str, password: str,
domain: str, port: int,
http_prefix: str,
box_name: str, page_number: int,
debug: bool, signing_priv_key_pem: str) -> {}:
"""C2S Authenticated GET of posts for a timeline
"""
if not session:
print('WARN: No session for c2s_box_json')
return None
domain_full = get_full_domain(domain, port)
actor = local_actor_url(http_prefix, nickname, domain_full)
auth_header = create_basic_auth_header(nickname, password)
profile_str = 'https://www.w3.org/ns/activitystreams'
headers = {
'host': domain,
'Content-type': 'application/json',
'Authorization': auth_header,
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
# GET json
url = actor + '/' + box_name + '?page=' + str(page_number)
box_json = get_json(signing_priv_key_pem, session, url, headers, None,
debug, __version__, http_prefix, None)
if get_json_valid(box_json) and debug:
print('DEBUG: GET c2s_box_json success')
return box_json
def seconds_between_published(published1: str, published2: str) -> int:
"""Returns the number of seconds between two published dates
"""
published1_time = \
date_from_string_format(published1, ['%Y-%m-%dT%H:%M:%S%z'])
if not published1_time:
print('EX: seconds_between_published unable to parse date 1 ' +
str(published1))
return -1
published2_time = \
date_from_string_format(published2, ['%Y-%m-%dT%H:%M:%S%z'])
if not published2_time:
print('EX: seconds_between_published unable to parse date 2 ' +
str(published2))
return -1
return (published2_time - published1_time).total_seconds()
def edited_post_filename(base_dir: str, nickname: str, domain: str,
post_json_object: {}, debug: bool,
max_time_diff_seconds: int,
system_language: str) -> (str, {}):
"""Returns the filename of the edited post
"""
if not has_object_dict(post_json_object):
return '', None
if not post_json_object.get('type'):
return '', None
if not post_json_object['object'].get('type'):
return '', None
if not post_json_object['object'].get('published'):
return '', None
if not post_json_object['object'].get('id'):
return '', None
if 'content' not in post_json_object['object']:
return '', None
if not post_json_object['object'].get('attributedTo'):
return '', None
if not get_attributed_to(post_json_object['object']['attributedTo']):
return '', None
actor = get_attributed_to(post_json_object['object']['attributedTo'])
actor_filename = \
acct_dir(base_dir, nickname, domain) + '/lastpost/' + \
actor.replace('/', '#')
if not os.path.isfile(actor_filename):
return '', None
post_id = remove_id_ending(post_json_object['object']['id'])
lastpost_id = None
try:
with open(actor_filename, 'r',
encoding='utf-8') as fp_actor:
lastpost_id = fp_actor.read()
except OSError:
print('EX: edited_post_filename unable to read ' + actor_filename)
return '', None
if not lastpost_id:
return '', None
if lastpost_id == post_id:
return '', None
lastpost_filename = \
locate_post(base_dir, nickname, domain, lastpost_id, False)
if not lastpost_filename:
return '', None
lastpost_json = load_json(lastpost_filename, 0)
if not lastpost_json:
return '', None
if not lastpost_json.get('type'):
return '', None
if lastpost_json['type'] != post_json_object['type']:
return '', None
if not lastpost_json.get('object'):
return '', None
if not isinstance(lastpost_json['object'], dict):
return '', None
if not lastpost_json['object'].get('type'):
return '', None
if lastpost_json['object']['type'] != post_json_object['object']['type']:
return '', None
if not lastpost_json['object'].get('published'):
return '', None
if not lastpost_json['object'].get('id'):
return '', None
if 'content' not in lastpost_json['object']:
return '', None
if not lastpost_json['object'].get('attributedTo'):
return '', None
if not get_attributed_to(lastpost_json['object']['attributedTo']):
return '', None
time_diff_seconds = \
seconds_between_published(lastpost_json['object']['published'],
post_json_object['object']['published'])
if time_diff_seconds > max_time_diff_seconds:
return '', None
if debug:
print(post_id + ' might be an edit of ' + lastpost_id)
lastpost_content = lastpost_json['object']['content']
if 'contentMap' in lastpost_json['object']:
if lastpost_json['object']['contentMap'].get(system_language):
lastpost_content = \
lastpost_json['object']['contentMap'][system_language]
content = post_json_object['object']['content']
if 'contentMap' in post_json_object['object']:
if post_json_object['object']['contentMap'].get(system_language):
content = \
post_json_object['object']['contentMap'][system_language]
if words_similarity(lastpost_content, content, 10) < 70:
return '', None
print(post_id + ' is an edit of ' + lastpost_id)
return lastpost_filename, lastpost_json
def get_original_post_from_announce_url(announce_url: str, base_dir: str,
nickname: str,
domain: str) -> (str, str, str):
"""From the url of an announce this returns the actor, url and
filename (if available) of the original post being announced
"""
post_filename = locate_post(base_dir, nickname, domain, announce_url)
if not post_filename:
return None, None, None
announce_post_json = load_json(post_filename, 0, 1)
if not announce_post_json:
return None, None, post_filename
if not announce_post_json.get('type'):
return None, None, post_filename
if announce_post_json['type'] != 'Announce':
return None, None, post_filename
if not announce_post_json.get('object'):
return None, None, post_filename
if not isinstance(announce_post_json['object'], str):
return None, None, post_filename
actor = url = None
# do we have the original post?
orig_post_id = announce_post_json['object']
orig_filename = locate_post(base_dir, nickname, domain, orig_post_id)
if orig_filename:
# we have the original post
orig_post_json = load_json(orig_filename, 0, 1)
if orig_post_json:
if has_object_dict(orig_post_json):
if orig_post_json['object'].get('attributedTo'):
attrib_field = \
orig_post_json['object']['attributedTo']
attrib = get_attributed_to(attrib_field)
if attrib:
actor = attrib
url = orig_post_id
elif orig_post_json['object'].get('actor'):
actor = get_actor_from_post(orig_post_json)
url = orig_post_id
else:
# we don't have the original post
if has_users_path(orig_post_id):
# get the actor from the original post url
orig_nick = get_nickname_from_actor(orig_post_id)
orig_domain, _ = get_domain_from_actor(orig_post_id)
if orig_nick and orig_domain:
actor = \
orig_post_id.split('/' + orig_nick + '/')[0] + \
'/' + orig_nick
url = orig_post_id
return actor, url, orig_filename
def get_max_profile_posts(base_dir: str, nickname: str, domain: str,
max_recent_posts: int) -> int:
"""Returns the maximum number of posts to show on the profile screen
"""
max_posts_filename = \
acct_dir(base_dir, nickname, domain) + '/max_profile_posts.txt'
max_profile_posts = 4
if not os.path.isfile(max_posts_filename):
return max_profile_posts
try:
with open(max_posts_filename, 'r', encoding='utf-8') as fp_posts:
max_posts_str = fp_posts.read()
if max_posts_str:
if max_posts_str.isdigit():
max_profile_posts = int(max_posts_str)
except OSError:
print('EX: unable to read maximum profile posts ' +
max_posts_filename)
if max_profile_posts < 1:
max_profile_posts = 1
if max_profile_posts > max_recent_posts:
max_profile_posts = max_recent_posts
return max_profile_posts
def set_max_profile_posts(base_dir: str, nickname: str, domain: str,
max_recent_posts: int) -> bool:
"""Sets the maximum number of posts to show on the profile screen
"""
max_posts_filename = \
acct_dir(base_dir, nickname, domain) + '/max_profile_posts.txt'
max_recent_posts_str = str(max_recent_posts)
try:
with open(max_posts_filename, 'w+',
encoding='utf-8') as fp_posts:
fp_posts.write(max_recent_posts_str)
except OSError:
print('EX: unable to save maximum profile posts ' +
max_posts_filename)
return False
return True