__filename__ = "posts.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.3.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "ActivityPub"
import json
import html
import datetime
import os
import shutil
import sys
import time
import random
from socket import error as SocketError
from time import gmtime, strftime
from collections import OrderedDict
from threads import thread_with_trace
from cache import store_person_in_cache
from cache import get_person_from_cache
from cache import expire_person_cache
from pprint import pprint
from session import create_session
from session import get_json
from session import post_json
from session import post_json_string
from session import post_image
from webfinger import webfinger_handle
from httpsig import create_signed_header
from siteactive import site_is_active
from languages import understood_post_language
from utils import valid_hash_tag
from utils import get_audio_extensions
from utils import get_summary_from_post
from utils import get_user_paths
from utils import invalid_ciphertext
from utils import has_object_string_type
from utils import remove_id_ending
from utils import replace_users_with_at
from utils import has_group_type
from utils import get_base_content_from_post
from utils import remove_domain_port
from utils import get_port_from_domain
from utils import has_object_dict
from utils import reject_post_id
from utils import remove_invalid_chars
from utils import file_last_modified
from utils import is_public_post
from utils import has_users_path
from utils import valid_post_date
from utils import get_full_domain
from utils import get_followers_list
from utils import is_evil
from utils import get_status_number
from utils import create_person_dir
from utils import url_permitted
from utils import get_nickname_from_actor
from utils import get_domain_from_actor
from utils import delete_post
from utils import valid_nickname
from utils import locate_post
from utils import load_json
from utils import save_json
from utils import get_config_param
from utils import locate_news_votes
from utils import locate_news_arrival
from utils import votes_on_newswire_item
from utils import remove_html
from utils import dangerous_markup
from utils import acct_dir
from utils import local_actor_url
from media import get_music_metadata
from media import attach_media
from media import replace_you_tube
from media import replace_twitter
from content import words_similarity
from content import limit_repeated_words
from content import post_tag_exists
from content import remove_long_words
from content import add_html_tags
from content import replace_emoji_from_tags
from content import remove_text_formatting
from auth import create_basic_auth_header
from blocking import is_blocked_hashtag
from blocking import is_blocked
from blocking import is_blocked_domain
from filters import is_filtered
from git import convert_post_to_patch
from linked_data_sig import generate_json_signature
from petnames import resolve_petnames
from video import convert_video_to_note
from context import get_individual_post_context
from maps import geocoords_from_map_link
def is_moderator(base_dir: str, nickname: str) -> bool:
"""Returns true if the given nickname is a moderator
"""
moderators_file = base_dir + '/accounts/moderators.txt'
if not os.path.isfile(moderators_file):
admin_name = get_config_param(base_dir, 'admin')
if not admin_name:
return False
if admin_name == nickname:
return True
return False
with open(moderators_file, 'r') as fp_mod:
lines = fp_mod.readlines()
if len(lines) == 0:
admin_name = get_config_param(base_dir, 'admin')
if not admin_name:
return False
if admin_name == nickname:
return True
for moderator in lines:
moderator = moderator.strip('\n').strip('\r')
if moderator == nickname:
return True
return False
def no_of_followers_on_domain(base_dir: str, handle: str,
domain: str, follow_file='followers.txt') -> int:
"""Returns the number of followers of the given handle from the given domain
"""
filename = base_dir + '/accounts/' + handle + '/' + follow_file
if not os.path.isfile(filename):
return 0
ctr = 0
with open(filename, 'r') as followers_file:
for follower_handle in followers_file:
if '@' in follower_handle:
follower_domain = follower_handle.split('@')[1]
follower_domain = follower_domain.replace('\n', '')
follower_domain = follower_domain.replace('\r', '')
if domain == follower_domain:
ctr += 1
return ctr
def _get_local_private_key(base_dir: str, nickname: str, domain: str) -> str:
"""Returns the private key for a local account
"""
if not domain or not nickname:
return None
handle = nickname + '@' + domain
key_filename = base_dir + '/keys/private/' + handle.lower() + '.key'
if not os.path.isfile(key_filename):
return None
with open(key_filename, 'r') as pem_file:
return pem_file.read()
return None
def get_instance_actor_key(base_dir: str, domain: str) -> str:
"""Returns the private key for the instance actor used for
signing GET posts
"""
return _get_local_private_key(base_dir, 'inbox', domain)
def _get_local_public_key(base_dir: str, nickname: str, domain: str) -> str:
"""Returns the public key for a local account
"""
if not domain or not nickname:
return None
handle = nickname + '@' + domain
key_filename = base_dir + '/keys/public/' + handle.lower() + '.key'
if not os.path.isfile(key_filename):
return None
with open(key_filename, 'r') as pem_file:
return pem_file.read()
return None
def _get_person_key(nickname: str, domain: str, base_dir: str,
key_type: str = 'public', debug: bool = False):
"""Returns the public or private key of a person
"""
if key_type == 'private':
key_pem = _get_local_private_key(base_dir, nickname, domain)
else:
key_pem = _get_local_public_key(base_dir, nickname, domain)
if not key_pem:
if debug:
print('DEBUG: ' + key_type + ' key file not found')
return ''
if len(key_pem) < 20:
if debug:
print('DEBUG: private key was too short: ' + key_pem)
return ''
return key_pem
def _clean_html(raw_html: str) -> str:
# text=BeautifulSoup(raw_html, 'html.parser').get_text()
text = raw_html
return html.unescape(text)
def get_user_url(wf_request: {}, source_id: int, debug: bool) -> str:
"""Gets the actor url from a webfinger request
"""
if not wf_request.get('links'):
if source_id == 72367:
print('get_user_url ' + str(source_id) +
' failed to get display name for webfinger ' +
str(wf_request))
else:
print('get_user_url webfinger activity+json contains no links ' +
str(source_id) + ' ' + str(wf_request))
return None
for link in wf_request['links']:
if not (link.get('type') and link.get('href')):
continue
if link['type'] != 'application/activity+json':
continue
if '/@' not in link['href']:
if debug and not has_users_path(link['href']):
print('get_user_url webfinger activity+json ' +
'contains single user instance actor ' +
str(source_id) + ' ' + str(link))
else:
return link['href'].replace('/@', '/users/')
return link['href']
return None
def parse_user_feed(signing_priv_key_pem: str,
session, feed_url: str, as_header: {},
project_version: str, http_prefix: str,
origin_domain: str, debug: bool, depth: int = 0) -> []:
if depth > 10:
if debug:
print('Maximum search depth reached')
return None
if debug:
print('Getting user feed for ' + feed_url)
print('User feed header ' + str(as_header))
print('http_prefix ' + str(http_prefix))
print('origin_domain ' + str(origin_domain))
feed_json = \
get_json(signing_priv_key_pem, session, feed_url, as_header, None,
debug, project_version, http_prefix, origin_domain)
if not feed_json:
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = 'application/ld+json; profile="' + profile_str + '"'
if as_header['Accept'] != accept_str:
as_header = {
'Accept': accept_str
}
feed_json = get_json(signing_priv_key_pem, session, feed_url,
as_header, None, debug, project_version,
http_prefix, origin_domain)
if not feed_json:
if debug:
print('No user feed was returned')
return None
if debug:
print('User feed:')
pprint(feed_json)
if 'orderedItems' in feed_json:
return feed_json['orderedItems']
if 'items' in feed_json:
return feed_json['items']
next_url = None
if 'first' in feed_json:
next_url = feed_json['first']
elif 'next' in feed_json:
next_url = feed_json['next']
if debug:
print('User feed next url: ' + str(next_url))
if next_url:
if isinstance(next_url, str):
if '?max_id=0' not in next_url:
user_feed = \
parse_user_feed(signing_priv_key_pem,
session, next_url, as_header,
project_version, http_prefix,
origin_domain, debug, depth + 1)
if user_feed:
return user_feed
elif isinstance(next_url, dict):
user_feed = next_url
if user_feed.get('orderedItems'):
return user_feed['orderedItems']
if user_feed.get('items'):
return user_feed['items']
return None
def _get_person_box_actor(session, base_dir: str, actor: str,
profile_str: str, as_header: {},
debug: bool, project_version: str,
http_prefix: str, origin_domain: str,
person_cache: {},
signing_priv_key_pem: str,
source_id: int) -> {}:
"""Returns the actor json for the given actor url
"""
person_json = \
get_person_from_cache(base_dir, actor, person_cache, True)
if person_json:
return person_json
if '/channel/' in actor or '/accounts/' in actor:
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
person_json = \
get_json(signing_priv_key_pem, session, actor, as_header, None,
debug, project_version, http_prefix, origin_domain)
if person_json:
return person_json
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
person_json = \
get_json(signing_priv_key_pem, session, actor, as_header, None,
debug, project_version, http_prefix, origin_domain)
if person_json:
return person_json
print('Unable to get actor for ' + actor + ' ' + str(source_id))
if not signing_priv_key_pem:
print('No signing key provided when getting actor')
return None
def get_person_box(signing_priv_key_pem: str, origin_domain: str,
base_dir: str, session, wf_request: {}, person_cache: {},
project_version: str, http_prefix: str,
nickname: str, domain: str,
box_name: str = 'inbox',
source_id=0) -> (str, str, str, str, str, str, str, bool):
debug = False
profile_str = 'https://www.w3.org/ns/activitystreams'
as_header = {
'Accept': 'application/activity+json; profile="' + profile_str + '"'
}
if not wf_request:
print('No webfinger given')
return None, None, None, None, None, None, None, None
# get the actor / person_url
if not wf_request.get('errors'):
# get the actor url from webfinger links
person_url = get_user_url(wf_request, source_id, debug)
else:
if nickname == 'dev':
# try single user instance
print('get_person_box: Trying single user instance with ld+json')
person_url = http_prefix + '://' + domain
as_header = {
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
else:
# the final fallback is a mastodon style url
person_url = local_actor_url(http_prefix, nickname, domain)
if not person_url:
return None, None, None, None, None, None, None, None
# get the actor json from the url
person_json = \
_get_person_box_actor(session, base_dir, person_url,
profile_str, as_header,
debug, project_version,
http_prefix, origin_domain,
person_cache, signing_priv_key_pem,
source_id)
if not person_json:
return None, None, None, None, None, None, None, None
is_group = False
if person_json.get('type'):
if person_json['type'] == 'Group':
is_group = True
# get the url for the box/collection
box_json = None
if not person_json.get(box_name):
if person_json.get('endpoints'):
if person_json['endpoints'].get(box_name):
box_json = person_json['endpoints'][box_name]
else:
box_json = person_json[box_name]
if not box_json:
return None, None, None, None, None, None, None, None
person_id = None
if person_json.get('id'):
person_id = person_json['id']
pub_key_id = None
pub_key = None
if person_json.get('publicKey'):
if person_json['publicKey'].get('id'):
pub_key_id = person_json['publicKey']['id']
if person_json['publicKey'].get('publicKeyPem'):
pub_key = person_json['publicKey']['publicKeyPem']
shared_inbox = None
if person_json.get('sharedInbox'):
shared_inbox = person_json['sharedInbox']
else:
if person_json.get('endpoints'):
if person_json['endpoints'].get('sharedInbox'):
shared_inbox = person_json['endpoints']['sharedInbox']
avatar_url = None
if person_json.get('icon'):
if person_json['icon'].get('url'):
avatar_url = person_json['icon']['url']
display_name = None
if person_json.get('name'):
display_name = person_json['name']
if dangerous_markup(person_json['name'], False):
display_name = '*ADVERSARY*'
elif is_filtered(base_dir,
nickname, domain,
display_name):
display_name = '*FILTERED*'
# have they moved?
if person_json.get('movedTo'):
display_name += ' ⌂'
store_person_in_cache(base_dir, person_url, person_json,
person_cache, True)
return box_json, pub_key_id, pub_key, person_id, shared_inbox, \
avatar_url, display_name, is_group
def _is_public_feed_post(item: {}, person_posts: {}, debug: bool) -> bool:
"""Is the given post a public feed post?
"""
if not isinstance(item, dict):
if debug:
print('item object is not a dict')
pprint(item)
return False
if not item.get('id'):
if debug:
print('No id')
return False
if not item.get('type'):
if debug:
print('No type')
return False
if item['type'] != 'Create' and \
item['type'] != 'Announce' and \
item['type'] != 'Page' and \
item['type'] != 'Note':
if debug:
print('Not a Create/Note/Announce type')
return False
if item.get('object'):
if isinstance(item['object'], dict):
if not item['object'].get('published'):
if debug:
print('No published attribute')
return False
elif isinstance(item['object'], str):
if not item.get('published'):
if debug:
print('No published attribute')
return False
else:
if debug:
print('object is not a dict or string')
return False
elif item['type'] == 'Note' or item['type'] == 'Page':
if not item.get('published'):
if debug:
print('No published attribute')
return False
if not person_posts.get(item['id']):
this_item = item
if item.get('object'):
this_item = item['object']
# check that this is a public post
# #Public should appear in the "to" list
item_is_note = False
if item['type'] == 'Note' or item['type'] == 'Page':
item_is_note = True
if isinstance(this_item, dict):
if this_item.get('to'):
is_public = False
for recipient in this_item['to']:
if recipient.endswith('#Public'):
is_public = True
break
if not is_public:
return False
elif isinstance(this_item, str) or item_is_note:
if item.get('to'):
is_public = False
for recipient in item['to']:
if recipient.endswith('#Public'):
is_public = True
break
if not is_public:
return False
return True
def is_create_inside_announce(item: {}) -> bool:
""" is this a Create inside of an Announce?
eg. lemmy feed item
"""
if not isinstance(item, dict):
return False
if item['type'] != 'Announce':
return False
if not item.get('object'):
return False
if not isinstance(item['object'], dict):
return False
if not item['object'].get('type'):
return False
if item['object']['type'] != 'Create':
return False
return True
def _get_posts(session, outbox_url: str, max_posts: int,
max_mentions: int,
max_emoji: int, max_attachments: int,
federation_list: [],
person_cache: {}, raw: bool,
simple: bool, debug: bool,
project_version: str, http_prefix: str,
origin_domain: str, system_language: str,
signing_priv_key_pem: str) -> {}:
"""Gets public posts from an outbox
"""
if debug:
print('Getting outbox posts for ' + outbox_url)
person_posts = {}
if not outbox_url:
return person_posts
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if raw:
if debug:
print('Returning the raw feed')
result = []
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix,
origin_domain, debug)
for item in user_feed:
result.append(item)
i += 1
if i == max_posts:
break
pprint(result)
return None
if debug:
print('Returning a human readable version of the feed')
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix,
origin_domain, debug)
if not user_feed:
return person_posts
i = 0
for item in user_feed:
if is_create_inside_announce(item):
item = item['object']
if not _is_public_feed_post(item, person_posts, debug):
continue
this_item = item
this_item_type = item['type']
if this_item_type not in ('Note', 'Page'):
this_item = item['object']
content = get_base_content_from_post(item, system_language)
content = content.replace(''', "'")
mentions = []
emoji = {}
summary = ''
in_reply_to = ''
attachment = []
sensitive = False
if isinstance(this_item, dict):
if this_item.get('tag'):
for tag_item in this_item['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'emoji':
if tag_item.get('name') and tag_item.get('icon'):
if tag_item['icon'].get('url'):
# No emoji from non-permitted domains
if url_permitted(tag_item['icon']['url'],
federation_list):
emoji_name = tag_item['name']
emoji_icon = tag_item['icon']['url']
emoji[emoji_name] = emoji_icon
else:
if debug:
print('url not permitted ' +
tag_item['icon']['url'])
if tag_type == 'mention':
if tag_item.get('name'):
if tag_item['name'] not in mentions:
mentions.append(tag_item['name'])
if len(mentions) > max_mentions:
if debug:
print('max mentions reached')
continue
if len(emoji) > max_emoji:
if debug:
print('max emojis reached')
continue
if this_item.get('summaryMap'):
if this_item['summaryMap'].get(system_language):
summary = this_item['summaryMap'][system_language]
if not summary and this_item.get('summary'):
if this_item['summary']:
summary = this_item['summary']
if this_item.get('inReplyTo'):
if this_item['inReplyTo']:
if isinstance(this_item['inReplyTo'], str):
# No replies to non-permitted domains
if not url_permitted(this_item['inReplyTo'],
federation_list):
if debug:
print('url not permitted ' +
this_item['inReplyTo'])
continue
in_reply_to = this_item['inReplyTo']
if this_item.get('attachment'):
if this_item['attachment']:
for attach in this_item['attachment']:
if attach.get('name') and attach.get('url'):
# no attachments from non-permitted domains
if url_permitted(attach['url'],
federation_list):
attachment.append([attach['name'],
attach['url']])
else:
if debug:
print('url not permitted ' +
attach['url'])
sensitive = False
if this_item.get('sensitive'):
sensitive = this_item['sensitive']
if content:
if simple:
print(_clean_html(content) + '\n')
else:
pprint(item)
person_posts[item['id']] = {
"sensitive": sensitive,
"inreplyto": in_reply_to,
"summary": summary,
"html": content,
"plaintext": _clean_html(content),
"attachment": attachment,
"mentions": mentions,
"emoji": emoji
}
i += 1
if i == max_posts:
break
return person_posts
def _get_common_words() -> str:
"""Returns a list of common words
"""
return (
'that', 'some', 'about', 'then', 'they', 'were',
'also', 'from', 'with', 'this', 'have', 'more',
'need', 'here', 'would', 'these', 'into', 'very',
'well', 'when', 'what', 'your', 'there', 'which',
'even', 'there', 'such', 'just', 'those', 'only',
'will', 'much', 'than', 'them', 'each', 'goes',
'been', 'over', 'their', 'where', 'could', 'though',
'like', 'think', 'same', 'maybe', 'really', 'thing',
'something', 'possible', 'actual', 'actually',
'because', 'around', 'having', 'especially', 'other',
'making', 'made', 'make', 'makes', 'including',
'includes', 'know', 'knowing', 'knows', 'things',
'say', 'says', 'saying', 'many', 'somewhat',
'problem', 'problems', 'idea', 'ideas',
'using', 'uses', 'https', 'still', 'want', 'wants'
)
def _update_word_frequency(content: str, word_frequency: {}) -> None:
"""Creates a dictionary containing words and the number of times
that they appear
"""
plain_text = remove_html(content)
remove_chars = ('.', ';', '?', '\n', ':')
for char in remove_chars:
plain_text = plain_text.replace(char, ' ')
words_list = plain_text.split(' ')
common_words = _get_common_words()
for word in words_list:
wordLen = len(word)
if wordLen < 3:
continue
if wordLen < 4:
if word.upper() != word:
continue
if '&' in word or \
'"' in word or \
'@' in word or \
"'" in word or \
"--" in word or \
'//' in word:
continue
if word.lower() in common_words:
continue
if word_frequency.get(word):
word_frequency[word] += 1
else:
word_frequency[word] = 1
def get_post_domains(session, outbox_url: str, max_posts: int,
max_mentions: int,
max_emoji: int, max_attachments: int,
federation_list: [],
person_cache: {},
debug: bool,
project_version: str, http_prefix: str,
domain: str,
word_frequency: {},
domain_list: [], system_language: str,
signing_priv_key_pem: str) -> []:
"""Returns a list of domains referenced within public posts
"""
if not outbox_url:
return []
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
post_domains = domain_list
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix, domain, debug)
for item in user_feed:
i += 1
if i > max_posts:
break
if not has_object_dict(item):
continue
content_str = get_base_content_from_post(item, system_language)
if content_str:
_update_word_frequency(content_str, word_frequency)
if item['object'].get('inReplyTo'):
if isinstance(item['object']['inReplyTo'], str):
post_domain, post_port = \
get_domain_from_actor(item['object']['inReplyTo'])
if post_domain not in post_domains:
post_domains.append(post_domain)
if item['object'].get('tag'):
for tag_item in item['object']['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'mention':
if tag_item.get('href'):
post_domain, post_port = \
get_domain_from_actor(tag_item['href'])
if post_domain not in post_domains:
post_domains.append(post_domain)
return post_domains
def _get_posts_for_blocked_domains(base_dir: str,
session, outbox_url: str, max_posts: int,
max_mentions: int,
max_emoji: int, max_attachments: int,
federation_list: [],
person_cache: {},
debug: bool,
project_version: str, http_prefix: str,
domain: str,
signing_priv_key_pem: str) -> {}:
"""Returns a dictionary of posts for blocked domains
"""
if not outbox_url:
return {}
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/outbox/' in outbox_url:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
blocked_posts = {}
i = 0
user_feed = parse_user_feed(signing_priv_key_pem,
session, outbox_url, as_header,
project_version, http_prefix, domain, debug)
for item in user_feed:
i += 1
if i > max_posts:
break
if not has_object_dict(item):
continue
if item['object'].get('inReplyTo'):
if isinstance(item['object']['inReplyTo'], str):
post_domain, post_port = \
get_domain_from_actor(item['object']['inReplyTo'])
if is_blocked_domain(base_dir, post_domain):
if item['object'].get('url'):
url = item['object']['url']
else:
url = item['object']['id']
if not blocked_posts.get(post_domain):
blocked_posts[post_domain] = [url]
else:
if url not in blocked_posts[post_domain]:
blocked_posts[post_domain].append(url)
if item['object'].get('tag'):
for tag_item in item['object']['tag']:
if not tag_item.get('type'):
continue
tag_type = tag_item['type'].lower()
if tag_type == 'mention' and tag_item.get('href'):
post_domain, post_port = \
get_domain_from_actor(tag_item['href'])
if is_blocked_domain(base_dir, post_domain):
if item['object'].get('url'):
url = item['object']['url']
else:
url = item['object']['id']
if not blocked_posts.get(post_domain):
blocked_posts[post_domain] = [url]
else:
if url not in blocked_posts[post_domain]:
blocked_posts[post_domain].append(url)
return blocked_posts
def delete_all_posts(base_dir: str,
nickname: str, domain: str, boxname: str) -> None:
"""Deletes all posts for a person from inbox or outbox
"""
if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews'):
return
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
for delete_filename in os.scandir(box_dir):
delete_filename = delete_filename.name
file_path = os.path.join(box_dir, delete_filename)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path, ignore_errors=False, onerror=None)
except OSError as ex:
print('ERROR: delete_all_posts ' + str(ex))
def save_post_to_box(base_dir: str, http_prefix: str, post_id: str,
nickname: str, domain: str, post_json_object: {},
boxname: str) -> str:
"""Saves the give json to the give box
Returns the filename
"""
if boxname not in ('inbox', 'outbox', 'tlblogs', 'tlnews', 'scheduled'):
return None
original_domain = domain
domain = remove_domain_port(domain)
if not post_id:
status_number, _ = get_status_number()
post_id = \
local_actor_url(http_prefix, nickname, original_domain) + \
'/statuses/' + status_number
post_json_object['id'] = post_id + '/activity'
if has_object_dict(post_json_object):
post_json_object['object']['id'] = post_id
post_json_object['object']['atomUri'] = post_id
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
filename = box_dir + '/' + post_id.replace('/', '#') + '.json'
save_json(post_json_object, filename)
return filename
def _update_hashtags_index(base_dir: str, tag: {}, new_post_id: str) -> None:
"""Writes the post url for hashtags to a file
This allows posts for a hashtag to be quickly looked up
"""
if tag['type'] != 'Hashtag':
return
# create hashtags directory
tags_dir = base_dir + '/tags'
if not os.path.isdir(tags_dir):
os.mkdir(tags_dir)
tag_name = tag['name']
tags_filename = tags_dir + '/' + tag_name[1:] + '.txt'
tagline = new_post_id + '\n'
if not os.path.isfile(tags_filename):
# create a new tags index file
try:
with open(tags_filename, 'w+') as tags_file:
tags_file.write(tagline)
except OSError:
print('EX: _update_hashtags_index unable to write tags file ' +
tags_filename)
else:
# prepend to tags index file
if tagline not in open(tags_filename).read():
try:
with open(tags_filename, 'r+') as tags_file:
content = tags_file.read()
if tagline not in content:
tags_file.seek(0, 0)
tags_file.write(tagline + content)
except OSError as ex:
print('EX: Failed to write entry to tags file ' +
tags_filename + ' ' + str(ex))
def _add_schedule_post(base_dir: str, nickname: str, domain: str,
event_date_str: str, post_id: str) -> None:
"""Adds a scheduled post to the index
"""
handle = nickname + '@' + domain
schedule_index_filename = \
base_dir + '/accounts/' + handle + '/schedule.index'
index_str = event_date_str + ' ' + post_id.replace('/', '#')
if os.path.isfile(schedule_index_filename):
if index_str not in open(schedule_index_filename).read():
try:
with open(schedule_index_filename, 'r+') as schedule_file:
content = schedule_file.read()
if index_str + '\n' not in content:
schedule_file.seek(0, 0)
schedule_file.write(index_str + '\n' + content)
print('DEBUG: scheduled post added to index')
except OSError as ex:
print('EX: Failed to write entry to scheduled posts index ' +
schedule_index_filename + ' ' + str(ex))
else:
try:
with open(schedule_index_filename, 'w+') as schedule_file:
schedule_file.write(index_str + '\n')
except OSError as ex:
print('EX: Failed to write entry to scheduled posts index2 ' +
schedule_index_filename + ' ' + str(ex))
def valid_content_warning(cw: str) -> str:
"""Returns a validated content warning
"""
cw = remove_html(cw)
# hashtags within content warnings apparently cause a lot of trouble
# so remove them
if '#' in cw:
cw = cw.replace('#', '').replace(' ', ' ')
return remove_invalid_chars(cw)
def _load_auto_cw(base_dir: str, nickname: str, domain: str) -> []:
"""Loads automatic CWs file and returns a list containing
the lines of the file
"""
filename = acct_dir(base_dir, nickname, domain) + '/autocw.txt'
if not os.path.isfile(filename):
return []
try:
with open(filename, 'r') as fp_auto:
return fp_auto.readlines()
except OSError:
print('EX: unable to load auto cw file ' + filename)
return []
def _add_auto_cw(base_dir: str, nickname: str, domain: str,
subject: str, content: str) -> str:
"""Appends any automatic CW to the subject line
and returns the new subject line
"""
new_subject = subject
auto_cw_list = _load_auto_cw(base_dir, nickname, domain)
for cw_rule in auto_cw_list:
if '->' not in cw_rule:
continue
rulematch = cw_rule.split('->')[0].strip()
if rulematch not in content:
continue
cw_str = cw_rule.split('->')[1].strip()
if new_subject:
if cw_str not in new_subject:
new_subject += ', ' + cw_str
else:
new_subject = cw_str
return new_subject
def _create_post_cw_from_reply(base_dir: str, nickname: str, domain: str,
in_reply_to: str,
sensitive: bool, summary: str,
system_language: str,
languages_understood: []) -> (bool, str):
"""If this is a reply and the original post has a CW
then use the same CW
"""
if in_reply_to and not sensitive:
# locate the post which this is a reply to and check if
# it has a content warning. If it does then reproduce
# the same warning
reply_post_filename = \
locate_post(base_dir, nickname, domain, in_reply_to)
if reply_post_filename:
reply_to_json = load_json(reply_post_filename)
if reply_to_json:
if reply_to_json.get('object'):
if reply_to_json['object'].get('sensitive'):
if reply_to_json['object']['sensitive']:
sensitive = True
if reply_to_json['object'].get('summary'):
summary = \
get_summary_from_post(reply_to_json,
system_language,
languages_understood)
return sensitive, summary
def _create_post_s2s(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, content: str, status_number: str,
published: str, new_post_id: str, post_context: {},
to_recipients: [], to_cc: [], in_reply_to: str,
sensitive: bool, comments_enabled: bool,
tags: [], attach_image_filename: str,
media_type: str, image_description: str, city: str,
post_object_type: str, summary: str,
in_reply_to_atom_uri: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str) -> {}:
"""Creates a new server-to-server post
"""
actor_url = local_actor_url(http_prefix, nickname, domain)
id_str = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number + '/replies'
new_post_url = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
new_post_attributed_to = \
local_actor_url(http_prefix, nickname, domain)
if not conversation_id:
conversation_id = new_post_id
new_post = {
'@context': post_context,
'id': new_post_id + '/activity',
'type': 'Create',
'actor': actor_url,
'published': published,
'to': to_recipients,
'cc': to_cc,
'object': {
'id': new_post_id,
'conversation': conversation_id,
'type': post_object_type,
'summary': summary,
'inReplyTo': in_reply_to,
'published': published,
'url': new_post_url,
'attributedTo': new_post_attributed_to,
'to': to_recipients,
'cc': to_cc,
'sensitive': sensitive,
'atomUri': new_post_id,
'inReplyToAtomUri': in_reply_to_atom_uri,
'commentsEnabled': comments_enabled,
'rejectReplies': not comments_enabled,
'mediaType': 'text/html',
'content': content,
'contentMap': {
system_language: content
},
'attachment': [],
'tag': tags,
'replies': {
'id': id_str,
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'next': id_str + '?only_other_accounts=true&page=true',
'partOf': id_str,
'items': []
}
}
}
}
if attach_image_filename:
new_post['object'] = \
attach_media(base_dir, http_prefix, nickname, domain, port,
new_post['object'], attach_image_filename,
media_type, image_description, city, low_bandwidth,
content_license_url)
return new_post
def _create_post_c2s(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, content: str, status_number: str,
published: str, new_post_id: str, post_context: {},
to_recipients: [], to_cc: [], in_reply_to: str,
sensitive: bool, comments_enabled: bool,
tags: [], attach_image_filename: str,
media_type: str, image_description: str, city: str,
post_object_type: str, summary: str,
in_reply_to_atom_uri: str, system_language: str,
conversation_id: str, low_bandwidth: str,
content_license_url: str) -> {}:
"""Creates a new client-to-server post
"""
domain_full = get_full_domain(domain, port)
id_str = \
local_actor_url(http_prefix, nickname, domain_full) + \
'/statuses/' + status_number + '/replies'
new_post_url = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
if not conversation_id:
conversation_id = new_post_id
new_post = {
"@context": post_context,
'id': new_post_id,
'conversation': conversation_id,
'type': post_object_type,
'summary': summary,
'inReplyTo': in_reply_to,
'published': published,
'url': new_post_url,
'attributedTo': local_actor_url(http_prefix, nickname, domain_full),
'to': to_recipients,
'cc': to_cc,
'sensitive': sensitive,
'atomUri': new_post_id,
'inReplyToAtomUri': in_reply_to_atom_uri,
'commentsEnabled': comments_enabled,
'rejectReplies': not comments_enabled,
'mediaType': 'text/html',
'content': content,
'contentMap': {
system_language: content
},
'attachment': [],
'tag': tags,
'replies': {
'id': id_str,
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'next': id_str + '?only_other_accounts=true&page=true',
'partOf': id_str,
'items': []
}
}
}
if attach_image_filename:
new_post = \
attach_media(base_dir, http_prefix, nickname, domain, port,
new_post, attach_image_filename,
media_type, image_description, city, low_bandwidth,
content_license_url)
return new_post
def _create_post_place_and_time(event_date: str, end_date: str,
event_time: str, end_time: str,
summary: str, content: str,
schedule_post: bool,
event_uuid: str,
location: str,
tags: []) -> str:
"""Adds a place and time to the tags on a new post
"""
end_date_str = None
if end_date:
event_name = summary
if not event_name:
event_name = content
end_date_str = end_date
if end_time:
if end_time.endswith('Z'):
end_date_str = end_date + 'T' + end_time
else:
end_date_str = end_date + 'T' + end_time + \
':00' + strftime("%z", gmtime())
else:
end_date_str = end_date + 'T12:00:00Z'
# get the starting date and time
event_date_str = None
if event_date:
event_name = summary
if not event_name:
event_name = content
event_date_str = event_date
if event_time:
if event_time.endswith('Z'):
event_date_str = event_date + 'T' + event_time
else:
event_date_str = event_date + 'T' + event_time + \
':00' + strftime("%z", gmtime())
else:
event_date_str = event_date + 'T12:00:00Z'
if not end_date_str:
end_date_str = event_date_str
if not schedule_post and not event_uuid:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Event",
"name": event_name,
"startTime": event_date_str,
"endTime": end_date_str
})
if location and not event_uuid:
latitude = longitude = None
if '://' in location:
_, latitude, longitude = \
geocoords_from_map_link(location)
if latitude and longitude:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Place",
"name": location,
"latitude": latitude,
"longitude": longitude
})
else:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Place",
"name": location
})
return event_date_str
def _consolidate_actors_list(actors_list: []) -> None:
""" consolidate duplicated actors
https://domain/@nick gets merged with https://domain/users/nick
"""
possible_duplicate_actors = []
for cc_actor in actors_list:
if '/@' in cc_actor:
if cc_actor not in possible_duplicate_actors:
possible_duplicate_actors.append(cc_actor)
if possible_duplicate_actors:
u_paths = get_user_paths()
remove_actors = []
for cc_actor in possible_duplicate_actors:
for usr_path in u_paths:
cc_actor_full = cc_actor.replace('/@', usr_path)
if cc_actor_full in actors_list:
if cc_actor not in remove_actors:
remove_actors.append(cc_actor)
break
for cc_actor in remove_actors:
actors_list.remove(cc_actor)
def _create_post_mentions(cc_url: str, new_post: {},
to_recipients: [], tags: []) -> None:
"""Updates mentions for a new post
"""
if not cc_url:
return
if len(cc_url) == 0:
return
if new_post.get('object'):
if cc_url not in new_post['object']['cc']:
new_post['object']['cc'] = [cc_url] + new_post['object']['cc']
# if this is a public post then include any mentions in cc
to_cc = new_post['object']['cc']
if len(to_recipients) != 1:
return
if to_recipients[0].endswith('#Public') and \
cc_url.endswith('/followers'):
for tag in tags:
if tag['type'] != 'Mention':
continue
if tag['href'] not in to_cc:
new_post['object']['cc'].append(tag['href'])
_consolidate_actors_list(new_post['object']['cc'])
new_post['cc'] = new_post['object']['cc']
else:
if cc_url not in new_post['cc']:
new_post['cc'] = [cc_url] + new_post['cc']
_consolidate_actors_list(['cc'])
def _create_post_mod_report(base_dir: str,
is_moderation_report: bool, new_post: {},
new_post_id: str) -> None:
""" if this is a moderation report then add a status
"""
if not is_moderation_report:
return
# add status
if new_post.get('object'):
new_post['object']['moderationStatus'] = 'pending'
else:
new_post['moderationStatus'] = 'pending'
# save to index file
moderation_index_file = base_dir + '/accounts/moderation.txt'
try:
with open(moderation_index_file, 'a+') as mod_file:
mod_file.write(new_post_id + '\n')
except OSError:
print('EX: unable to write moderation index file ' +
moderation_index_file)
def get_actor_from_in_reply_to(in_reply_to: str) -> str:
"""Tries to get the replied to actor from the inReplyTo post id
Note: this will not always be successful for some instance types
"""
reply_nickname = get_nickname_from_actor(in_reply_to)
if not reply_nickname:
return None
reply_actor = None
if '/' + reply_nickname + '/' in in_reply_to:
reply_actor = \
in_reply_to.split('/' + reply_nickname + '/')[0] + \
'/' + reply_nickname
elif '#' + reply_nickname + '#' in in_reply_to:
reply_actor = \
in_reply_to.split('#' + reply_nickname + '#')[0] + \
'#' + reply_nickname
reply_actor = reply_actor.replace('#', '/')
if not reply_actor:
return None
if '://' not in reply_actor:
return None
return reply_actor
def _create_post_base(base_dir: str,
nickname: str, domain: str, port: int,
to_url: str, cc_url: str, http_prefix: str, content: str,
followers_only: bool, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str,
media_type: str, image_description: str, city: str,
is_moderation_report: bool,
is_article: bool,
in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str,
location: str,
event_uuid: str, category: str,
join_mode: str,
end_date: str, end_time: str,
maximum_attendee_capacity: int,
replies_moderation_option: str,
anonymous_participation_enabled: bool,
event_status: str, ticket_url: str,
system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Creates a message
"""
content = remove_invalid_chars(content)
subject = _add_auto_cw(base_dir, nickname, domain, subject, content)
if nickname != 'news':
mentioned_recipients = \
get_mentioned_people(base_dir, http_prefix, content, domain, False)
else:
mentioned_recipients = ''
# add hashtags from audio file ID3 tags, such as Artist, Album, etc
if attach_image_filename and media_type:
audio_types = get_audio_extensions()
music_metadata = None
for ext in audio_types:
if ext in media_type:
music_metadata = get_music_metadata(attach_image_filename)
break
if music_metadata:
for audio_tag, audio_value in music_metadata.items():
if audio_tag == 'title' or audio_tag == 'track':
continue
# capitalize and remove any spaces
audio_value = audio_value.title().replace(' ', '')
# check that the tag is valid
if valid_hash_tag(audio_value) and \
'#' + audio_value not in content:
# check that it hasn't been blocked
if not is_blocked_hashtag(base_dir, audio_value):
content += ' #' + audio_value
tags = []
hashtags_dict = {}
domain = get_full_domain(domain, port)
# add tags
if nickname != 'news':
content = \
add_html_tags(base_dir, http_prefix,
nickname, domain, content,
mentioned_recipients,
hashtags_dict, True)
# replace emoji with unicode
tags = []
for tag_name, tag in hashtags_dict.items():
tags.append(tag)
# get list of tags
if nickname != 'news':
content = \
replace_emoji_from_tags(None, base_dir, content, tags, 'content',
False, True)
# remove replaced emoji
hashtags_dict_copy = hashtags_dict.copy()
for tag_name, tag in hashtags_dict_copy.items():
if tag.get('name'):
if tag['name'].startswith(':'):
if tag['name'] not in content:
del hashtags_dict[tag_name]
status_number, published = get_status_number()
new_post_id = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
sensitive = False
summary = None
if subject:
summary = remove_invalid_chars(valid_content_warning(subject))
sensitive = True
to_recipients = []
to_cc = []
if to_url:
if not isinstance(to_url, str):
print('ERROR: to_url is not a string')
return None
to_recipients = [to_url]
# who to send to
if mentioned_recipients:
for mention in mentioned_recipients:
if mention not in to_cc:
to_cc.append(mention)
is_public = False
for recipient in to_recipients:
if recipient.endswith('#Public'):
is_public = True
break
# create a list of hashtags
# Only posts which are #Public are searchable by hashtag
if hashtags_dict:
for tag_name, tag in hashtags_dict.items():
if not post_tag_exists(tag['type'], tag['name'], tags):
tags.append(tag)
if is_public:
_update_hashtags_index(base_dir, tag, new_post_id)
# print('Content tags: ' + str(tags))
sensitive, summary = \
_create_post_cw_from_reply(base_dir, nickname, domain,
in_reply_to, sensitive, summary,
system_language, languages_understood)
event_date_str = \
_create_post_place_and_time(event_date, end_date,
event_time, end_time,
summary, content, schedule_post,
event_uuid, location, tags)
post_context = get_individual_post_context()
if not is_public:
# make sure that CC doesn't also contain a To address
# eg. To: [ "https://mydomain/users/foo/followers" ]
# CC: [ "X", "Y", "https://mydomain/users/foo", "Z" ]
remove_from_cc = []
for cc_recipient in to_cc:
for send_to_actor in to_recipients:
if cc_recipient in send_to_actor and \
cc_recipient not in remove_from_cc:
remove_from_cc.append(cc_recipient)
break
for cc_removal in remove_from_cc:
to_cc.remove(cc_removal)
else:
if in_reply_to:
# If this is a public post then get the actor being
# replied to end ensure that it is within the CC list
reply_actor = get_actor_from_in_reply_to(in_reply_to)
if reply_actor:
if reply_actor not in to_cc:
to_cc.append(reply_actor)
# the type of post to be made
post_object_type = 'Note'
if is_article:
post_object_type = 'Article'
if not client_to_server:
new_post = \
_create_post_s2s(base_dir, nickname, domain, port,
http_prefix, content, status_number,
published, new_post_id, post_context,
to_recipients, to_cc, in_reply_to,
sensitive, comments_enabled,
tags, attach_image_filename,
media_type, image_description, city,
post_object_type, summary,
in_reply_to_atom_uri, system_language,
conversation_id, low_bandwidth,
content_license_url)
else:
new_post = \
_create_post_c2s(base_dir, nickname, domain, port,
http_prefix, content, status_number,
published, new_post_id, post_context,
to_recipients, to_cc, in_reply_to,
sensitive, comments_enabled,
tags, attach_image_filename,
media_type, image_description, city,
post_object_type, summary,
in_reply_to_atom_uri, system_language,
conversation_id, low_bandwidth,
content_license_url)
_create_post_mentions(cc_url, new_post, to_recipients, tags)
_create_post_mod_report(base_dir, is_moderation_report,
new_post, new_post_id)
# If a patch has been posted - i.e. the output from
# git format-patch - then convert the activitypub type
convert_post_to_patch(base_dir, nickname, domain, new_post)
if schedule_post:
if event_date and event_time:
# add an item to the scheduled post index file
_add_schedule_post(base_dir, nickname, domain,
event_date_str, new_post_id)
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'scheduled')
else:
print('Unable to create scheduled post without ' +
'date and time values')
return new_post
elif save_to_file:
if is_article:
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'tlblogs')
else:
save_post_to_box(base_dir, http_prefix, new_post_id,
nickname, domain, new_post, 'outbox')
return new_post
def outbox_message_create_wrap(http_prefix: str,
nickname: str, domain: str, port: int,
message_json: {}) -> {}:
"""Wraps a received message in a Create
https://www.w3.org/TR/activitypub/#object-without-create
"""
domain = get_full_domain(domain, port)
status_number, published = get_status_number()
if message_json.get('published'):
published = message_json['published']
new_post_id = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
cc_list = []
if message_json.get('cc'):
cc_list = message_json['cc']
new_post = {
"@context": "https://www.w3.org/ns/activitystreams",
'id': new_post_id + '/activity',
'type': 'Create',
'actor': local_actor_url(http_prefix, nickname, domain),
'published': published,
'to': message_json['to'],
'cc': cc_list,
'object': message_json
}
new_post['object']['id'] = new_post['id']
new_post['object']['url'] = \
http_prefix + '://' + domain + '/@' + nickname + '/' + status_number
new_post['object']['atomUri'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/statuses/' + status_number
return new_post
def _post_is_addressed_to_followers(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
post_json_object: {}) -> bool:
"""Returns true if the given post is addressed to followers of the nickname
"""
domain_full = get_full_domain(domain, port)
if not post_json_object.get('object'):
return False
to_list = []
cc_list = []
if post_json_object['type'] != 'Update' and \
has_object_dict(post_json_object):
if post_json_object['object'].get('to'):
to_list = post_json_object['object']['to']
if post_json_object['object'].get('cc'):
cc_list = post_json_object['object']['cc']
else:
if post_json_object.get('to'):
to_list = post_json_object['to']
if post_json_object.get('cc'):
cc_list = post_json_object['cc']
followers_url = \
local_actor_url(http_prefix, nickname, domain_full) + '/followers'
# does the followers url exist in 'to' or 'cc' lists?
addressed_to_followers = False
if followers_url in to_list:
addressed_to_followers = True
elif followers_url in cc_list:
addressed_to_followers = True
return addressed_to_followers
def pin_post(base_dir: str, nickname: str, domain: str,
pinned_content: str, followers_only: bool) -> None:
"""Pins the given post Id to the profile of then given account
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
try:
with open(pinned_filename, 'w+') as pin_file:
pin_file.write(pinned_content)
except OSError:
print('EX: unable to write ' + pinned_filename)
def undo_pinned_post(base_dir: str, nickname: str, domain: str) -> None:
"""Removes pinned content for then given account
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
if not os.path.isfile(pinned_filename):
return
try:
os.remove(pinned_filename)
except OSError:
print('EX: undo_pinned_post unable to delete ' + pinned_filename)
def get_pinned_post_as_json(base_dir: str, http_prefix: str,
nickname: str, domain: str,
domain_full: str, system_language: str) -> {}:
"""Returns the pinned profile post as json
"""
account_dir = acct_dir(base_dir, nickname, domain)
pinned_filename = account_dir + '/pinToProfile.txt'
pinned_post_json = {}
actor = local_actor_url(http_prefix, nickname, domain_full)
if os.path.isfile(pinned_filename):
pinned_content = None
with open(pinned_filename, 'r') as pin_file:
pinned_content = pin_file.read()
if pinned_content:
pinned_post_json = {
'atomUri': actor + '/pinned',
'attachment': [],
'attributedTo': actor,
'cc': [
actor + '/followers'
],
'content': pinned_content,
'contentMap': {
system_language: pinned_content
},
'id': actor + '/pinned',
'inReplyTo': None,
'inReplyToAtomUri': None,
'published': file_last_modified(pinned_filename),
'replies': {},
'sensitive': False,
'summary': None,
'tag': [],
'to': ['https://www.w3.org/ns/activitystreams#Public'],
'type': 'Note',
'url': replace_users_with_at(actor) + '/pinned'
}
return pinned_post_json
def json_pin_post(base_dir: str, http_prefix: str,
nickname: str, domain: str,
domain_full: str, system_language: str) -> {}:
"""Returns a pinned post as json
"""
pinned_post_json = \
get_pinned_post_as_json(base_dir, http_prefix,
nickname, domain,
domain_full, system_language)
items_list = []
if pinned_post_json:
items_list = [pinned_post_json]
actor = local_actor_url(http_prefix, nickname, domain_full)
post_context = get_individual_post_context()
return {
'@context': post_context,
'id': actor + '/collections/featured',
'orderedItems': items_list,
'totalItems': len(items_list),
'type': 'OrderedCollection'
}
def regenerate_index_for_box(base_dir: str,
nickname: str, domain: str,
box_name: str) -> None:
"""Generates an index for the given box if it doesn't exist
Used by unit tests to artificially create an index
"""
box_dir = acct_dir(base_dir, nickname, domain) + '/' + box_name
box_index_filename = box_dir + '.index'
if not os.path.isdir(box_dir):
return
if os.path.isfile(box_index_filename):
return
index_lines = []
for _, _, files in os.walk(box_dir):
for fname in files:
if ':##' not in fname:
continue
index_lines.append(fname)
break
index_lines.sort(reverse=True)
result = ''
try:
with open(box_index_filename, 'w+') as fp_box:
for line in index_lines:
result += line + '\n'
fp_box.write(line + '\n')
except OSError:
print('EX: unable to generate index for ' + box_name + ' ' + result)
print('Index generated for ' + box_name + '\n' + result)
def create_public_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, followers_only: bool, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
in_reply_to: str,
in_reply_to_atom_uri: str, subject: str,
schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, is_article: bool, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Public post
"""
domain_full = get_full_domain(domain, port)
is_moderation_report = False
event_uuid = None
category = None
join_mode = None
end_date = event_date
end_time = event_end_time
maximum_attendee_capacity = None
replies_moderation_option = None
anonymous_participation_enabled = None
event_status = None
ticket_url = None
local_actor = local_actor_url(http_prefix, nickname, domain_full)
return _create_post_base(base_dir, nickname, domain, port,
'https://www.w3.org/ns/activitystreams#Public',
local_actor + '/followers',
http_prefix, content, followers_only,
save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
is_moderation_report, is_article,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
event_uuid, category, join_mode,
end_date, end_time,
maximum_attendee_capacity,
replies_moderation_option,
anonymous_participation_enabled,
event_status, ticket_url, system_language,
conversation_id, low_bandwidth,
content_license_url,
languages_understood)
def _append_citations_to_blog_post(base_dir: str,
nickname: str, domain: str,
blog_json: {}) -> None:
"""Appends any citations to a new blog post
"""
# append citations tags, stored in a file
citations_filename = \
acct_dir(base_dir, nickname, domain) + '/.citations.txt'
if not os.path.isfile(citations_filename):
return
citations_separator = '#####'
with open(citations_filename, 'r') as fp_cit:
citations = fp_cit.readlines()
for line in citations:
if citations_separator not in line:
continue
sections = line.strip().split(citations_separator)
if len(sections) != 3:
continue
# date_str = sections[0]
title = sections[1]
link = sections[2]
tag_json = {
"type": "Article",
"name": title,
"url": link
}
blog_json['object']['tag'].append(tag_json)
def create_blog_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, followers_only: bool, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
in_reply_to: str, in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
blog_json = \
create_public_post(base_dir,
nickname, domain, port, http_prefix,
content, followers_only, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, event_end_time, location,
True, system_language, conversation_id,
low_bandwidth, content_license_url,
languages_understood)
blog_json['object']['url'] = \
blog_json['object']['url'].replace('/@', '/users/')
_append_citations_to_blog_post(base_dir, nickname, domain, blog_json)
return blog_json
def create_news_post(base_dir: str,
domain: str, port: int, http_prefix: str,
content: str, followers_only: bool, save_to_file: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
subject: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
client_to_server = False
in_reply_to = None
in_reply_to_atom_uri = None
schedule_post = False
event_date = None
event_time = None
event_end_time = None
location = None
blog = \
create_public_post(base_dir,
'news', domain, port, http_prefix,
content, followers_only, save_to_file,
client_to_server, False,
attach_image_filename, media_type,
image_description, city,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post,
event_date, event_time, event_end_time, location,
True, system_language, conversation_id,
low_bandwidth, content_license_url,
languages_understood)
blog['object']['type'] = 'Article'
return blog
def create_question_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, q_options: [],
followers_only: bool, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
subject: str, durationDays: int,
system_language: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Question post with multiple choice options
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
message_json = \
_create_post_base(base_dir, nickname, domain, port,
'https://www.w3.org/ns/activitystreams#Public',
local_actor + '/followers',
http_prefix, content, followers_only, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, False, None, None, subject,
False, None, None, None, None, None,
None, None, None,
None, None, None, None, None, system_language,
None, low_bandwidth, content_license_url,
languages_understood)
message_json['object']['type'] = 'Question'
message_json['object']['oneOf'] = []
message_json['object']['votersCount'] = 0
curr_time = datetime.datetime.utcnow()
days_since_epoch = \
int((curr_time - datetime.datetime(1970, 1, 1)).days + durationDays)
end_time = datetime.datetime(1970, 1, 1) + \
datetime.timedelta(days_since_epoch)
message_json['object']['endTime'] = end_time.strftime("%Y-%m-%dT%H:%M:%SZ")
for question_option in q_options:
message_json['object']['oneOf'].append({
"type": "Note",
"name": question_option,
"replies": {
"type": "Collection",
"totalItems": 0
}
})
return message_json
def create_unlisted_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, followers_only: bool,
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
in_reply_to: str, in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str, event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Unlisted post. This has the #Public and followers links inverted.
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
return _create_post_base(base_dir, nickname, domain, port,
local_actor + '/followers',
'https://www.w3.org/ns/activitystreams#Public',
http_prefix, content, followers_only,
save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, False,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date,
event_time, location,
None, None, None, event_date, event_end_time,
None, None, None, None, None, system_language,
conversation_id, low_bandwidth,
content_license_url, languages_understood)
def create_followers_only_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, followers_only: bool,
save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, schedule_post: bool,
event_date: str,
event_time: str, event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Followers only post
"""
domain_full = get_full_domain(domain, port)
local_actor = local_actor_url(http_prefix, nickname, domain_full)
return _create_post_base(base_dir, nickname, domain, port,
local_actor + '/followers', None,
http_prefix, content, followers_only,
save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, False,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
None, None, None, event_date, event_end_time,
None, None, None, None, None, system_language,
conversation_id, low_bandwidth,
content_license_url, languages_understood)
def get_mentioned_people(base_dir: str, http_prefix: str,
content: str, domain: str, debug: bool) -> []:
"""Extracts a list of mentioned actors from the given message content
"""
if '@' not in content:
return None
mentions = []
words = content.split(' ')
for wrd in words:
if not wrd.startswith('@'):
continue
handle = wrd[1:]
if debug:
print('DEBUG: mentioned handle ' + handle)
if '@' not in handle:
handle = handle + '@' + domain
if not os.path.isdir(base_dir + '/accounts/' + handle):
continue
else:
external_domain = handle.split('@')[1]
if not ('.' in external_domain or
external_domain == 'localhost'):
continue
mentioned_nickname = handle.split('@')[0]
mentioned_domain = handle.split('@')[1].strip('\n').strip('\r')
if ':' in mentioned_domain:
mentioned_domain = remove_domain_port(mentioned_domain)
if not valid_nickname(mentioned_domain, mentioned_nickname):
continue
actor = \
local_actor_url(http_prefix, mentioned_nickname,
handle.split('@')[1])
mentions.append(actor)
return mentions
def create_direct_message_post(base_dir: str,
nickname: str, domain: str, port: int,
http_prefix: str,
content: str, followers_only: bool,
save_to_file: bool, client_to_server: bool,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
in_reply_to: str,
in_reply_to_atom_uri: str,
subject: str, debug: bool,
schedule_post: bool,
event_date: str, event_time: str,
event_end_time: str,
location: str, system_language: str,
conversation_id: str, low_bandwidth: bool,
content_license_url: str,
languages_understood: [],
dm_is_chat: bool) -> {}:
"""Direct Message post
"""
content = resolve_petnames(base_dir, nickname, domain, content)
mentioned_people = \
get_mentioned_people(base_dir, http_prefix, content, domain, debug)
if debug:
print('mentioned_people: ' + str(mentioned_people))
if not mentioned_people:
return None
post_to = None
post_cc = None
message_json = \
_create_post_base(base_dir, nickname, domain, port,
post_to, post_cc,
http_prefix, content, followers_only, save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, False,
in_reply_to, in_reply_to_atom_uri, subject,
schedule_post, event_date, event_time, location,
None, None, None, event_date, event_end_time,
None, None, None, None, None, system_language,
conversation_id, low_bandwidth,
content_license_url, languages_understood)
# mentioned recipients go into To rather than Cc
message_json['to'] = message_json['object']['cc']
message_json['object']['to'] = message_json['to']
message_json['cc'] = []
message_json['object']['cc'] = []
if dm_is_chat:
message_json['object']['type'] = 'ChatMessage'
if schedule_post:
post_id = remove_id_ending(message_json['object']['id'])
save_post_to_box(base_dir, http_prefix, post_id,
nickname, domain, message_json, 'scheduled')
return message_json
def create_report_post(base_dir: str,
nickname: str, domain: str, port: int, http_prefix: str,
content: str, followers_only: bool, save_to_file: bool,
client_to_server: bool, comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
debug: bool, subject: str, system_language: str,
low_bandwidth: bool,
content_license_url: str,
languages_understood: []) -> {}:
"""Send a report to moderators
"""
domain_full = get_full_domain(domain, port)
# add a title to distinguish moderation reports from other posts
report_title = 'Moderation Report'
if not subject:
subject = report_title
else:
if not subject.startswith(report_title):
subject = report_title + ': ' + subject
# create the list of moderators from the moderators file
moderators_list = []
moderators_file = base_dir + '/accounts/moderators.txt'
if os.path.isfile(moderators_file):
with open(moderators_file, 'r') as fp_mod:
for line in fp_mod:
line = line.strip('\n').strip('\r')
if line.startswith('#'):
continue
if line.startswith('/users/'):
line = line.replace('users', '')
if line.startswith('@'):
line = line[1:]
if '@' in line:
nick = line.split('@')[0]
moderator_actor = \
local_actor_url(http_prefix, nick, domain_full)
if moderator_actor not in moderators_list:
moderators_list.append(moderator_actor)
continue
if line.startswith('http') or \
line.startswith('ipfs') or \
line.startswith('ipns') or \
line.startswith('hyper'):
# must be a local address - no remote moderators
if '://' + domain_full + '/' in line:
if line not in moderators_list:
moderators_list.append(line)
else:
if '/' not in line:
moderator_actor = \
local_actor_url(http_prefix, line, domain_full)
if moderator_actor not in moderators_list:
moderators_list.append(moderator_actor)
if len(moderators_list) == 0:
# if there are no moderators then the admin becomes the moderator
admin_nickname = get_config_param(base_dir, 'admin')
if admin_nickname:
local_actor = \
local_actor_url(http_prefix, admin_nickname, domain_full)
moderators_list.append(local_actor)
if not moderators_list:
return None
if debug:
print('DEBUG: Sending report to moderators')
print(str(moderators_list))
post_to = moderators_list
post_cc = None
post_json_object = None
for to_url in post_to:
# who is this report going to?
to_nickname = to_url.split('/users/')[1]
handle = to_nickname + '@' + domain
post_json_object = \
_create_post_base(base_dir, nickname, domain, port,
to_url, post_cc,
http_prefix, content, followers_only,
save_to_file,
client_to_server, comments_enabled,
attach_image_filename, media_type,
image_description, city,
True, False, None, None, subject,
False, None, None, None, None, None,
None, None, None,
None, None, None, None, None, system_language,
None, low_bandwidth, content_license_url,
languages_understood)
if not post_json_object:
continue
# save a notification file so that the moderator
# knows something new has appeared
new_report_file = base_dir + '/accounts/' + handle + '/.newReport'
if os.path.isfile(new_report_file):
continue
try:
with open(new_report_file, 'w+') as fp_report:
fp_report.write(to_url + '/moderation')
except OSError:
print('EX: create_report_post unable to write ' + new_report_file)
return post_json_object
def thread_send_post(session, post_json_str: str, federation_list: [],
inbox_url: str, base_dir: str,
signature_header_json: {},
signature_header_json_ld: {},
post_log: [], debug: bool,
http_prefix: str, domain_full: str) -> None:
"""Sends a with retries
"""
tries = 0
send_interval_sec = 30
for _ in range(20):
post_result = None
unauthorized = False
if debug:
print('Getting post_json_string for ' + inbox_url)
try:
post_result, unauthorized, return_code = \
post_json_string(session, post_json_str, federation_list,
inbox_url, signature_header_json,
debug, http_prefix, domain_full)
if return_code >= 500 and return_code < 600:
# if an instance is returning a code which indicates that
# it might have a runtime error, like 503, then don't
# continue to post to it
break
if debug:
print('Obtained post_json_string for ' + inbox_url +
' unauthorized: ' + str(unauthorized))
except Exception as ex:
print('ERROR: post_json_string failed ' + str(ex))
if unauthorized:
# try again with application/ld+json header
post_result = None
unauthorized = False
if debug:
print('Getting ld post_json_string for ' + inbox_url)
try:
post_result, unauthorized, return_code = \
post_json_string(session, post_json_str, federation_list,
inbox_url, signature_header_json_ld,
debug, http_prefix, domain_full)
if return_code >= 500 and return_code < 600:
# if an instance is returning a code which indicates that
# it might have a runtime error, like 503, then don't
# continue to post to it
break
if debug:
print('Obtained ld post_json_string for ' + inbox_url +
' unauthorized: ' + str(unauthorized))
except Exception as ex:
print('ERROR: ld post_json_string failed ' + str(ex))
if unauthorized:
print('WARN: thread_send_post: Post is unauthorized ' +
inbox_url + ' ' + post_json_str)
break
if post_result:
log_str = 'Success on try ' + str(tries) + ': ' + post_json_str
else:
log_str = 'Retry ' + str(tries) + ': ' + post_json_str
post_log.append(log_str)
# keep the length of the log finite
# Don't accumulate massive files on systems with limited resources
while len(post_log) > 16:
post_log.pop(0)
if debug:
# save the log file
post_log_filename = base_dir + '/post.log'
if os.path.isfile(post_log_filename):
with open(post_log_filename, 'a+') as log_file:
log_file.write(log_str + '\n')
else:
with open(post_log_filename, 'w+') as log_file:
log_file.write(log_str + '\n')
if post_result:
if debug:
print('DEBUG: successful json post to ' + inbox_url)
# our work here is done
break
if debug:
print(post_json_str)
print('DEBUG: json post to ' + inbox_url +
' failed. Waiting for ' +
str(send_interval_sec) + ' seconds.')
time.sleep(send_interval_sec)
tries += 1
def send_post(signing_priv_key_pem: str, project_version: str,
session, base_dir: str, nickname: str, domain: str, port: int,
to_nickname: str, to_domain: str, to_port: int, cc: str,
http_prefix: str, content: str, followers_only: bool,
save_to_file: bool, client_to_server: bool,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
federation_list: [], send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
is_article: bool, system_language: str,
languages_understood: [],
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
low_bandwidth: bool, content_license_url: str,
debug: bool = False, in_reply_to: str = None,
in_reply_to_atom_uri: str = None, subject: str = None) -> int:
"""Post to another inbox. Used by unit tests.
"""
with_digest = True
conversation_id = None
if to_nickname == 'inbox':
# shared inbox actor on @domain@domain
to_nickname = to_domain
to_domain = get_full_domain(to_domain, to_port)
handle = http_prefix + '://' + to_domain + '/@' + to_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return 1
if not isinstance(wf_request, dict):
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return 1
if not client_to_server:
post_to_box = 'inbox'
else:
post_to_box = 'outbox'
if is_article:
post_to_box = 'tlblogs'
# get the actor inbox for the To handle
origin_domain = domain
(inbox_url, _, pub_key, to_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, post_to_box,
72533)
if not inbox_url:
return 3
if not pub_key:
return 4
if not to_person_id:
return 5
# shared_inbox is optional
post_json_object = \
_create_post_base(base_dir, nickname, domain, port,
to_person_id, cc, http_prefix, content,
followers_only, save_to_file, client_to_server,
comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, is_article, in_reply_to,
in_reply_to_atom_uri, subject,
False, None, None, None, None, None,
None, None, None,
None, None, None, None, None, system_language,
conversation_id, low_bandwidth,
content_license_url, languages_understood)
# get the senders private key
private_key_pem = _get_person_key(nickname, domain, base_dir, 'private')
if len(private_key_pem) == 0:
return 6
if to_domain not in inbox_url:
return 7
post_path = inbox_url.split(to_domain, 1)[1]
if not post_json_object.get('signature'):
try:
signed_post_json_object = post_json_object.copy()
generate_json_signature(signed_post_json_object, private_key_pem)
post_json_object = signed_post_json_object
except Exception as ex:
print('WARN: send_post failed to JSON-LD sign post, ' + str(ex))
pprint(signed_post_json_object)
# convert json to string so that there are no
# subsequent conversions after creating message body digest
post_json_str = json.dumps(post_json_object)
# construct the http header, including the message body digest
signature_header_json = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str, 'application/activity+json')
signature_header_json_ld = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str, 'application/ld+json')
# if the "to" domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
domain_full = get_full_domain(domain, port)
if to_domain in shared_items_federated_domains:
if shared_item_federation_tokens.get(domain_full):
signature_header_json['Origin'] = domain_full
signature_header_json_ld['Origin'] = domain_full
signature_header_json['SharesCatalog'] = \
shared_item_federation_tokens[domain_full]
signature_header_json_ld['SharesCatalog'] = \
shared_item_federation_tokens[domain_full]
if debug:
print('SharesCatalog added to header')
elif debug:
print(domain_full + ' not in shared_item_federation_tokens')
elif debug:
print(to_domain + ' not in shared_items_federated_domains ' +
str(shared_items_federated_domains))
if debug:
print('signature_header_json: ' + str(signature_header_json))
# Keep the number of threads being used small
while len(send_threads) > 1000:
print('WARN: Maximum threads reached - killing send thread')
send_threads[0].kill()
send_threads.pop(0)
print('WARN: thread killed')
print('THREAD: thread_send_post')
thr = \
thread_with_trace(target=thread_send_post,
args=(session,
post_json_str,
federation_list,
inbox_url, base_dir,
signature_header_json.copy(),
signature_header_json_ld.copy(),
post_log, debug, http_prefix,
domain_full), daemon=True)
send_threads.append(thr)
thr.start()
return 0
def send_post_via_server(signing_priv_key_pem: str, project_version: str,
base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
to_nickname: str, to_domain: str, to_port: int,
cc: str,
http_prefix: str, content: str, followers_only: bool,
comments_enabled: bool,
attach_image_filename: str, media_type: str,
image_description: str, city: str,
cached_webfingers: {}, person_cache: {},
is_article: bool, system_language: str,
languages_understood: [],
low_bandwidth: bool,
content_license_url: str,
event_date: str, event_time: str, event_end_time: str,
location: str,
debug: bool = False,
in_reply_to: str = None,
in_reply_to_atom_uri: str = None,
conversation_id: str = None,
subject: str = None) -> int:
"""Send a post via a proxy (c2s)
"""
if not session:
print('WARN: No session for send_post_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
from_domain_full, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: post webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: post webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
if is_article:
post_to_box = 'tlblogs'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain_full, post_to_box,
82796)
if not inbox_url:
if debug:
print('DEBUG: post no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: post no actor was found for ' + handle)
return 4
# Get the json for the c2s post, not saving anything to file
# Note that base_dir is set to None
save_to_file = False
client_to_server = True
if to_domain.lower().endswith('public'):
to_person_id = 'https://www.w3.org/ns/activitystreams#Public'
cc = local_actor_url(http_prefix, from_nickname, from_domain_full) + \
'/followers'
else:
if to_domain.lower().endswith('followers') or \
to_domain.lower().endswith('followersonly'):
to_person_id = \
local_actor_url(http_prefix,
from_nickname, from_domain_full) + \
'/followers'
else:
to_domain_full = get_full_domain(to_domain, to_port)
to_person_id = \
local_actor_url(http_prefix, to_nickname, to_domain_full)
post_json_object = \
_create_post_base(base_dir,
from_nickname, from_domain, from_port,
to_person_id, cc, http_prefix, content,
followers_only, save_to_file, client_to_server,
comments_enabled,
attach_image_filename, media_type,
image_description, city,
False, is_article, in_reply_to,
in_reply_to_atom_uri, subject,
False,
event_date, event_time, location,
None, None, None, event_date, event_end_time,
None, None, None, None, None, system_language,
conversation_id, low_bandwidth,
content_license_url, languages_understood)
auth_header = create_basic_auth_header(from_nickname, password)
if attach_image_filename:
headers = {
'host': from_domain_full,
'Authorization': auth_header
}
post_result = \
post_image(session, attach_image_filename, [],
inbox_url, headers, http_prefix, from_domain_full)
if not post_result:
if debug:
print('DEBUG: post failed to upload image')
# return 9
headers = {
'host': from_domain_full,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_dumps = json.dumps(post_json_object)
post_result, unauthorized, return_code = \
post_json_string(session, post_dumps, [],
inbox_url, headers, debug,
http_prefix, from_domain_full,
5, True)
if not post_result:
if debug:
if unauthorized:
print('DEBUG: POST failed for c2s to ' +
inbox_url + ' unathorized')
else:
print('DEBUG: POST failed for c2s to ' +
inbox_url + ' return code ' + str(return_code))
return 5
if debug:
print('DEBUG: c2s POST success')
return 0
def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}:
"""Returns a dictionary with followers grouped by domain
"""
handle = nickname + '@' + domain
followers_filename = base_dir + '/accounts/' + handle + '/followers.txt'
if not os.path.isfile(followers_filename):
return None
grouped = {}
with open(followers_filename, 'r') as foll_file:
for follower_handle in foll_file:
if '@' not in follower_handle:
continue
fhandle = \
follower_handle.strip().replace('\n', '').replace('\r', '')
follower_domain = fhandle.split('@')[1]
if not grouped.get(follower_domain):
grouped[follower_domain] = [fhandle]
else:
grouped[follower_domain].append(fhandle)
return grouped
def _add_followers_to_public_post(post_json_object: {}) -> None:
"""Adds followers entry to cc if it doesn't exist
"""
if not post_json_object.get('actor'):
return
if isinstance(post_json_object['object'], str):
if not post_json_object.get('to'):
return
if len(post_json_object['to']) > 1:
return
if len(post_json_object['to']) == 0:
return
if not post_json_object['to'][0].endswith('#Public'):
return
if post_json_object.get('cc'):
return
post_json_object['cc'] = post_json_object['actor'] + '/followers'
elif has_object_dict(post_json_object):
if not post_json_object['object'].get('to'):
return
if len(post_json_object['object']['to']) > 1:
return
if len(post_json_object['object']['to']) == 0:
return
if not post_json_object['object']['to'][0].endswith('#Public'):
return
if post_json_object['object'].get('cc'):
return
post_json_object['object']['cc'] = \
post_json_object['actor'] + '/followers'
def send_signed_json(post_json_object: {}, session, base_dir: str,
nickname: str, domain: str, port: int,
to_nickname: str, to_domain: str, to_port: int, cc: str,
http_prefix: str, save_to_file: bool,
client_to_server: bool, federation_list: [],
send_threads: [], post_log: [], cached_webfingers: {},
person_cache: {}, debug: bool, project_version: str,
shared_items_token: str, group_account: bool,
signing_priv_key_pem: str,
source_id: int, curr_domain: str,
onion_domain: str, i2p_domain: str) -> int:
"""Sends a signed json object to an inbox/outbox
"""
if debug:
print('DEBUG: send_signed_json start')
if not session:
print('WARN: No session specified for send_signed_json')
return 8
with_digest = True
if to_domain.endswith('.onion') or to_domain.endswith('.i2p'):
http_prefix = 'http'
if to_nickname == 'inbox':
# shared inbox actor on @domain@domain
to_nickname = to_domain
to_domain = get_full_domain(to_domain, to_port)
to_domain_url = http_prefix + '://' + to_domain
if not site_is_active(to_domain_url, 10):
print('Domain is inactive: ' + to_domain_url)
return 9
print('Domain is active: ' + to_domain_url)
handle_base = to_domain_url + '/@'
if to_nickname:
handle = handle_base + to_nickname
else:
single_user_instance_nickname = 'dev'
handle = handle_base + single_user_instance_nickname
if debug:
print('DEBUG: handle - ' + handle + ' to_port ' + str(to_port))
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
domain, project_version, debug,
group_account, signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: webfinger for ' + handle + ' failed')
return 1
if not isinstance(wf_request, dict):
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return 1
if wf_request.get('errors'):
if debug:
print('DEBUG: webfinger for ' + handle +
' failed with errors ' + str(wf_request['errors']))
if not client_to_server:
post_to_box = 'inbox'
else:
post_to_box = 'outbox'
# get the actor inbox/outbox for the To handle
origin_domain = domain
(inbox_url, _, pub_key, to_person_id, shared_inbox_url, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, post_to_box,
source_id)
print("inbox_url: " + str(inbox_url))
print("to_person_id: " + str(to_person_id))
print("shared_inbox_url: " + str(shared_inbox_url))
if inbox_url:
if inbox_url.endswith('/actor/inbox') or \
inbox_url.endswith('/instance.actor/inbox'):
inbox_url = shared_inbox_url
if not inbox_url:
if debug:
print('DEBUG: missing inbox_url')
return 3
if debug:
print('DEBUG: Sending to endpoint ' + inbox_url)
if not pub_key:
if debug:
print('DEBUG: missing pubkey')
return 4
if not to_person_id:
if debug:
print('DEBUG: missing person_id')
return 5
# shared_inbox is optional
# get the senders private key
account_domain = origin_domain
if onion_domain:
if account_domain == onion_domain:
account_domain = curr_domain
if i2p_domain:
if account_domain == i2p_domain:
account_domain = curr_domain
private_key_pem = \
_get_person_key(nickname, account_domain, base_dir, 'private', debug)
if len(private_key_pem) == 0:
if debug:
print('DEBUG: Private key not found for ' +
nickname + '@' + account_domain +
' in ' + base_dir + '/keys/private')
return 6
if to_domain not in inbox_url:
if debug:
print('DEBUG: ' + to_domain + ' is not in ' + inbox_url)
return 7
post_path = inbox_url.split(to_domain, 1)[1]
_add_followers_to_public_post(post_json_object)
if not post_json_object.get('signature'):
try:
signed_post_json_object = post_json_object.copy()
generate_json_signature(signed_post_json_object, private_key_pem)
post_json_object = signed_post_json_object
except BaseException as ex:
print('WARN: send_signed_json failed to JSON-LD sign post, ' +
str(ex))
pprint(signed_post_json_object)
# convert json to string so that there are no
# subsequent conversions after creating message body digest
post_json_str = json.dumps(post_json_object)
# if the sender domain has changed from clearnet to onion or i2p
# then change the content of the post accordingly
if debug:
print('Checking for changed origin domain: ' +
domain + ' ' + curr_domain)
if domain != curr_domain:
if not curr_domain.endswith('.onion') and \
not curr_domain.endswith('.i2p'):
if debug:
print('Changing post content sender domain from ' +
curr_domain + ' to ' + domain)
post_json_str = \
post_json_str.replace(curr_domain, domain)
# construct the http header, including the message body digest
signature_header_json = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str,
'application/activity+json')
signature_header_json_ld = \
create_signed_header(None, private_key_pem, nickname, domain, port,
to_domain, to_port,
post_path, http_prefix, with_digest,
post_json_str,
'application/ld+json')
# optionally add a token so that the receiving instance may access
# your shared items catalog
if shared_items_token:
signature_header_json['Origin'] = get_full_domain(domain, port)
signature_header_json['SharesCatalog'] = shared_items_token
elif debug:
print('Not sending shared items federation token')
# Keep the number of threads being used small
while len(send_threads) > 1000:
print('WARN: Maximum threads reached - killing send thread')
send_threads[0].kill()
send_threads.pop(0)
print('WARN: thread killed')
if debug:
print('DEBUG: starting thread to send post')
pprint(post_json_object)
domain_full = get_full_domain(domain, port)
print('THREAD: thread_send_post 2')
thr = \
thread_with_trace(target=thread_send_post,
args=(session,
post_json_str,
federation_list,
inbox_url, base_dir,
signature_header_json.copy(),
signature_header_json_ld.copy(),
post_log, debug,
http_prefix, domain_full), daemon=True)
send_threads.append(thr)
# thr.start()
return 0
def add_to_field(activity_type: str, post_json_object: {},
debug: bool) -> ({}, bool):
"""The Follow/Add/Remove activity doesn't have a 'to' field and so one
needs to be added so that activity distribution happens in a consistent way
Returns true if a 'to' field exists or was added
"""
if post_json_object.get('to'):
return post_json_object, True
if debug:
pprint(post_json_object)
print('DEBUG: no "to" field when sending to named addresses 2')
is_same_type = False
to_field_added = False
if post_json_object.get('object'):
if isinstance(post_json_object['object'], str):
if post_json_object.get('type'):
if post_json_object['type'] == activity_type:
is_same_type = True
if debug:
print('DEBUG: ' +
'add_to_field1 "to" field assigned to ' +
activity_type)
to_address = post_json_object['object']
if '/statuses/' in to_address:
to_address = to_address.split('/statuses/')[0]
post_json_object['to'] = [to_address]
if debug:
print('DEBUG: "to" address added: ' + to_address)
to_field_added = True
elif has_object_dict(post_json_object):
# add a to field to bookmark add or remove
if post_json_object.get('type') and \
post_json_object.get('actor') and \
post_json_object['object'].get('type'):
if post_json_object['type'] == 'Add' or \
post_json_object['type'] == 'Remove':
if post_json_object['object']['type'] == 'Document':
post_json_object['to'] = \
[post_json_object['actor']]
post_json_object['object']['to'] = \
[post_json_object['actor']]
to_field_added = True
if not to_field_added and \
post_json_object['object'].get('type'):
if post_json_object['object']['type'] == activity_type:
is_same_type = True
if isinstance(post_json_object['object']['object'], str):
if debug:
print('DEBUG: add_to_field2 ' +
'"to" field assigned to ' +
activity_type)
to_address = post_json_object['object']['object']
if '/statuses/' in to_address:
to_address = to_address.split('/statuses/')[0]
post_json_object['object']['to'] = [to_address]
post_json_object['to'] = [to_address]
if debug:
print('DEBUG: "to" address added: ' + to_address)
to_field_added = True
if not is_same_type:
return post_json_object, True
if to_field_added:
return post_json_object, True
return post_json_object, False
def _is_profile_update(post_json_object: {}) -> bool:
"""Is the given post a profile update?
for actor updates there is no 'to' within the object
"""
if post_json_object.get('type'):
if has_object_string_type(post_json_object, False):
if (post_json_object['type'] == 'Update' and
(post_json_object['object']['type'] == 'Person' or
post_json_object['object']['type'] == 'Application' or
post_json_object['object']['type'] == 'Group' or
post_json_object['object']['type'] == 'Service')):
return True
return False
def _send_to_named_addresses(server, session, session_onion, session_i2p,
base_dir: str,
nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str) -> None:
"""sends a post to the specific named addresses in to/cc
"""
if not session:
print('WARN: No session for sendToNamedAddresses')
return
if not post_json_object.get('object'):
return
is_profile_update = False
if has_object_dict(post_json_object):
if _is_profile_update(post_json_object):
# use the original object, which has a 'to'
recipients_object = post_json_object
is_profile_update = True
if not is_profile_update:
if not post_json_object['object'].get('to'):
if debug:
pprint(post_json_object)
print('DEBUG: ' +
'no "to" field when sending to named addresses')
if has_object_string_type(post_json_object, debug):
if post_json_object['object']['type'] == 'Follow' or \
post_json_object['object']['type'] == 'Join':
post_json_obj2 = post_json_object['object']['object']
if isinstance(post_json_obj2, str):
if debug:
print('DEBUG: _send_to_named_addresses ' +
'"to" field assigned to Follow')
post_json_object['object']['to'] = \
[post_json_object['object']['object']]
if not post_json_object['object'].get('to'):
return
recipients_object = post_json_object['object']
else:
post_json_object, field_added = \
add_to_field('Follow', post_json_object, debug)
if not field_added:
return
post_json_object, field_added = \
add_to_field('Like', post_json_object, debug)
if not field_added:
return
recipients_object = post_json_object
recipients = []
recipient_type = ('to', 'cc')
for rtype in recipient_type:
if not recipients_object.get(rtype):
continue
if isinstance(recipients_object[rtype], list):
if debug:
pprint(recipients_object)
print('recipients_object: ' + str(recipients_object[rtype]))
for address in recipients_object[rtype]:
if not address:
continue
if '/' not in address:
continue
if address.endswith('#Public'):
continue
if address.endswith('/followers'):
continue
recipients.append(address)
elif isinstance(recipients_object[rtype], str):
address = recipients_object[rtype]
if address:
if '/' in address:
if address.endswith('#Public'):
continue
if address.endswith('/followers'):
continue
recipients.append(address)
if not recipients:
if debug:
print('DEBUG: no individual recipients')
return
if debug:
print('DEBUG: Sending individually addressed posts: ' +
str(recipients))
# randomize the recipients list order, so that we are not favoring
# any particular account in terms of delivery time
random.shuffle(recipients)
# this is after the message has arrived at the server
client_to_server = False
for address in recipients:
to_nickname = get_nickname_from_actor(address)
if not to_nickname:
continue
to_domain, to_port = get_domain_from_actor(address)
if not to_domain:
continue
# Don't send profile/actor updates to yourself
if is_profile_update:
domain_full = get_full_domain(domain, port)
to_domain_full = get_full_domain(to_domain, to_port)
if nickname == to_nickname and \
domain_full == to_domain_full:
if debug:
print('Not sending profile update to self. ' +
nickname + '@' + domain_full)
continue
# if we have an alt onion domain and we are sending to
# another onion domain then switch the clearnet
# domain for the onion one
from_domain = domain
from_domain_full = get_full_domain(domain, port)
from_http_prefix = http_prefix
curr_session = session
curr_proxy_type = proxy_type
session_type = 'default'
if onion_domain:
if not from_domain.endswith('.onion') and \
to_domain.endswith('.onion'):
from_domain = onion_domain
from_domain_full = onion_domain
from_http_prefix = 'http'
curr_session = session_onion
port = 80
to_port = 80
curr_proxy_type = 'tor'
session_type = 'tor'
if i2p_domain:
if not from_domain.endswith('.i2p') and \
to_domain.endswith('.i2p'):
from_domain = i2p_domain
from_domain_full = i2p_domain
from_http_prefix = 'http'
curr_session = session_i2p
port = 80
to_port = 80
curr_proxy_type = 'i2p'
session_type = 'i2p'
cc_list = []
if debug:
to_domain_full = get_full_domain(to_domain, to_port)
print('DEBUG: Post sending s2s: ' +
nickname + '@' + from_domain_full +
' to ' + to_nickname + '@' + to_domain_full)
# if the "to" domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
shared_items_token = None
if to_domain in shared_items_federated_domains:
if shared_item_federation_tokens.get(from_domain_full):
shared_items_token = \
shared_item_federation_tokens[from_domain_full]
group_account = has_group_type(base_dir, address, person_cache)
if not curr_session:
curr_session = create_session(curr_proxy_type)
if server:
if session_type == 'tor':
server.session_onion = curr_session
elif session_type == 'i2p':
server.session_i2p = curr_session
else:
server.session = curr_session
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
cc_list, from_http_prefix, True, client_to_server,
federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 34436782,
domain, onion_domain, i2p_domain)
def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str,
i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str):
"""Returns a thread used to send a post to named addresses
"""
print('THREAD: _send_to_named_addresses')
send_thread = \
thread_with_trace(target=_send_to_named_addresses,
args=(server, session, session_onion, session_i2p,
base_dir, nickname, domain,
onion_domain, i2p_domain, port,
http_prefix, federation_list,
send_threads, post_log,
cached_webfingers, person_cache,
post_json_object, debug,
project_version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem,
proxy_type), daemon=True)
try:
send_thread.start()
except SocketError as ex:
print('WARN: socket error while starting ' +
'thread to send to named addresses. ' + str(ex))
return None
except ValueError as ex:
print('WARN: error while starting ' +
'thread to send to named addresses. ' + str(ex))
return None
return send_thread
def _has_shared_inbox(session, http_prefix: str, domain: str,
debug: bool, signing_priv_key_pem: str) -> bool:
"""Returns true if the given domain has a shared inbox
This tries the new and the old way of webfingering the shared inbox
"""
try_handles = []
if ':' not in domain:
try_handles.append(domain + '@' + domain)
try_handles.append('inbox@' + domain)
for handle in try_handles:
wf_request = webfinger_handle(session, handle, http_prefix, {},
domain, __version__, debug, False,
signing_priv_key_pem)
if wf_request:
if isinstance(wf_request, dict):
if not wf_request.get('errors'):
return True
return False
def _sending_profile_update(post_json_object: {}) -> bool:
"""Returns true if the given json is a profile update
"""
if post_json_object['type'] != 'Update':
return False
if not has_object_string_type(post_json_object, False):
return False
activity_type = post_json_object['object']['type']
if activity_type in ('Person', 'Application', 'Group', 'Service'):
return True
return False
def send_to_followers(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str) -> None:
"""sends a post to the followers of the given nickname
"""
print('send_to_followers')
if not _post_is_addressed_to_followers(base_dir, nickname, domain,
port, http_prefix,
post_json_object):
if debug:
print('Post is not addressed to followers')
return
print('Post is addressed to followers')
grouped = group_followers_by_domain(base_dir, nickname, domain)
if not grouped:
if debug:
print('Post to followers did not resolve any domains')
return
print('Post to followers resolved domains')
# print(str(grouped))
# this is after the message has arrived at the server
client_to_server = False
curr_proxy_type = None
if domain.endswith('.onion'):
curr_proxy_type = 'tor'
elif domain.endswith('.i2p'):
curr_proxy_type = 'i2p'
sending_start_time = datetime.datetime.utcnow()
print('Sending post to followers begins ' +
sending_start_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
sending_ctr = 0
# randomize the order of sending to instances
randomized_instances = []
for follower_domain, follower_handles in grouped.items():
randomized_instances.append([follower_domain, follower_handles])
random.shuffle(randomized_instances)
# send out to each instance
for group_send in randomized_instances:
follower_domain = group_send[0]
follower_handles = group_send[1]
print('Sending post to followers progress ' +
str(int(sending_ctr * 100 / len(grouped.items()))) + '% ' +
follower_domain)
sending_ctr += 1
if debug:
pprint(follower_handles)
# if the followers domain is within the shared items
# federation list then send the token for this domain
# so that it can request a catalog
shared_items_token = None
if follower_domain in shared_items_federated_domains:
domain_full = get_full_domain(domain, port)
if shared_item_federation_tokens.get(domain_full):
shared_items_token = shared_item_federation_tokens[domain_full]
# check that the follower's domain is active
follower_domain_url = http_prefix + '://' + follower_domain
if not site_is_active(follower_domain_url, 10):
print('Sending post to followers domain is inactive: ' +
follower_domain_url)
continue
print('Sending post to followers domain is active: ' +
follower_domain_url)
# select the appropriate session
curr_session = session
curr_http_prefix = http_prefix
if onion_domain:
if follower_domain.endswith('.onion'):
curr_session = session_onion
curr_http_prefix = 'http'
if i2p_domain:
if follower_domain.endswith('.i2p'):
curr_session = session_i2p
curr_http_prefix = 'http'
with_shared_inbox = \
_has_shared_inbox(curr_session, curr_http_prefix, follower_domain,
debug, signing_priv_key_pem)
if debug:
if with_shared_inbox:
print(follower_domain + ' has shared inbox')
if not with_shared_inbox:
print('Sending post to followers, ' + follower_domain +
' does not have a shared inbox')
to_port = port
index = 0
to_domain = follower_handles[index].split('@')[1]
if ':' in to_domain:
to_port = get_port_from_domain(to_domain)
to_domain = remove_domain_port(to_domain)
cc_list = ''
# if we are sending to an onion domain and we
# have an alt onion domain then use the alt
from_domain = domain
from_http_prefix = http_prefix
session_type = 'default'
if onion_domain:
if to_domain.endswith('.onion'):
from_domain = onion_domain
from_http_prefix = 'http'
port = 80
to_port = 80
curr_proxy_type = 'tor'
session_type = 'tor'
if i2p_domain:
if to_domain.endswith('.i2p'):
from_domain = i2p_domain
from_http_prefix = 'http'
port = 80
to_port = 80
curr_proxy_type = 'i2p'
session_type = 'i2p'
if not curr_session:
curr_session = create_session(curr_proxy_type)
if server:
if session_type == 'tor':
server.session_onion = curr_session
elif session_type == 'i2p':
server.session_i2p = curr_session
else:
server.session = curr_session
if with_shared_inbox:
to_nickname = follower_handles[index].split('@')[0]
group_account = False
if to_nickname.startswith('!'):
group_account = True
to_nickname = to_nickname[1:]
# if there are more than one followers on the domain
# then send the post to the shared inbox
if len(follower_handles) > 1:
to_nickname = 'inbox'
if to_nickname != 'inbox' and post_json_object.get('type'):
if _sending_profile_update(post_json_object):
print('Sending post to followers ' +
'shared inbox of ' + to_domain)
to_nickname = 'inbox'
print('Sending post to followers from ' +
nickname + '@' + domain +
' to ' + to_nickname + '@' + to_domain)
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
cc_list, from_http_prefix, True,
client_to_server, federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 639342,
domain, onion_domain, i2p_domain)
else:
# randomize the order of handles, so that we are not
# favoring any particular account in terms of its delivery time
random.shuffle(follower_handles)
# send to individual followers without using a shared inbox
for handle in follower_handles:
print('Sending post to followers ' + handle)
to_nickname = handle.split('@')[0]
group_account = False
if to_nickname.startswith('!'):
group_account = True
to_nickname = to_nickname[1:]
if post_json_object['type'] != 'Update':
print('Sending post to followers from ' +
nickname + '@' + domain + ' to ' +
to_nickname + '@' + to_domain)
else:
print('Sending post to followers profile update from ' +
nickname + '@' + domain + ' to ' +
to_nickname + '@' + to_domain)
send_signed_json(post_json_object, curr_session, base_dir,
nickname, from_domain, port,
to_nickname, to_domain, to_port,
cc_list, from_http_prefix, True,
client_to_server, federation_list,
send_threads, post_log, cached_webfingers,
person_cache, debug, project_version,
shared_items_token, group_account,
signing_priv_key_pem, 634219,
domain, onion_domain, i2p_domain)
time.sleep(4)
if debug:
print('DEBUG: End of send_to_followers')
sending_end_time = datetime.datetime.utcnow()
sending_mins = \
int((sending_end_time - sending_start_time).total_seconds() / 60)
print('Sending post to followers ends ' + str(sending_mins) + ' mins')
def send_to_followers_thread(server, session, session_onion, session_i2p,
base_dir: str, nickname: str, domain: str,
onion_domain: str, i2p_domain: str, port: int,
http_prefix: str, federation_list: [],
send_threads: [], post_log: [],
cached_webfingers: {}, person_cache: {},
post_json_object: {}, debug: bool,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str):
"""Returns a thread used to send a post to followers
"""
print('THREAD: send_to_followers')
send_thread = \
thread_with_trace(target=send_to_followers,
args=(server, session, session_onion, session_i2p,
base_dir, nickname, domain,
onion_domain, i2p_domain, port,
http_prefix, federation_list,
send_threads, post_log,
cached_webfingers, person_cache,
post_json_object.copy(), debug,
project_version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem), daemon=True)
try:
send_thread.start()
except SocketError as ex:
print('WARN: socket error while starting ' +
'thread to send to followers. ' + str(ex))
return None
except ValueError as ex:
print('WARN: error while starting ' +
'thread to send to followers. ' + str(ex))
return None
return send_thread
def create_inbox(recent_posts_cache: {},
session, base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, items_per_page: int, header_only: bool,
page_number: int) -> {}:
return _create_box_indexed(recent_posts_cache,
session, base_dir, 'inbox',
nickname, domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number)
def create_bookmarks_timeline(session, base_dir: str,
nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'tlbookmarks',
nickname, domain,
port, http_prefix, items_per_page, header_only,
True, 0, False, 0, page_number)
def create_dm_timeline(recent_posts_cache: {},
session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed(recent_posts_cache,
session, base_dir, 'dm', nickname,
domain, port, http_prefix, items_per_page,
header_only, True, 0, False, 0, page_number)
def create_replies_timeline(recent_posts_cache: {},
session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed(recent_posts_cache, session,
base_dir, 'tlreplies',
nickname, domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number)
def create_blogs_timeline(session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'tlblogs', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number)
def create_features_timeline(session, base_dir: str,
nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'tlfeatures', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number)
def create_media_timeline(session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'tlmedia', nickname,
domain, port, http_prefix,
items_per_page, header_only, True,
0, False, 0, page_number)
def create_news_timeline(session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str, items_per_page: int,
header_only: bool, newswire_votes_threshold: int,
positive_voting: bool, voting_time_mins: int,
page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'outbox', 'news',
domain, port, http_prefix,
items_per_page, header_only, True,
newswire_votes_threshold, positive_voting,
voting_time_mins, page_number)
def create_outbox(session, base_dir: str, nickname: str, domain: str,
port: int, http_prefix: str,
items_per_page: int, header_only: bool, authorized: bool,
page_number: int) -> {}:
return _create_box_indexed({}, session, base_dir, 'outbox',
nickname, domain, port, http_prefix,
items_per_page, header_only, authorized,
0, False, 0, page_number)
def create_moderation(base_dir: str, nickname: str, domain: str, port: int,
http_prefix: str, items_per_page: int, header_only: bool,
page_number: int) -> {}:
box_dir = create_person_dir(nickname, domain, base_dir, 'inbox')
boxname = 'moderation'
domain = get_full_domain(domain, port)
if not page_number:
page_number = 1
page_str = '?page=' + str(page_number)
box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
box_header = {
'@context': 'https://www.w3.org/ns/activitystreams',
'first': box_url + '?page=true',
'id': box_url,
'last': box_url + '?page=true',
'totalItems': 0,
'type': 'OrderedCollection'
}
box_items = {
'@context': 'https://www.w3.org/ns/activitystreams',
'id': box_url + page_str,
'orderedItems': [
],
'partOf': box_url,
'type': 'OrderedCollectionPage'
}
if is_moderator(base_dir, nickname):
moderation_index_file = base_dir + '/accounts/moderation.txt'
if os.path.isfile(moderation_index_file):
with open(moderation_index_file, 'r') as index_file:
lines = index_file.readlines()
box_header['totalItems'] = len(lines)
if header_only:
return box_header
page_lines = []
if len(lines) > 0:
end_line_number = \
len(lines) - 1 - int(items_per_page * page_number)
if end_line_number < 0:
end_line_number = 0
start_line_number = \
len(lines) - 1 - int(items_per_page * (page_number - 1))
if start_line_number < 0:
start_line_number = 0
line_number = start_line_number
while line_number >= end_line_number:
line_no_str = lines[line_number].strip('\n').strip('\r')
page_lines.append(line_no_str)
line_number -= 1
for post_url in page_lines:
post_filename = \
box_dir + '/' + post_url.replace('/', '#') + '.json'
if os.path.isfile(post_filename):
post_json_object = load_json(post_filename)
if post_json_object:
box_items['orderedItems'].append(post_json_object)
if header_only:
return box_header
return box_items
def is_image_media(session, base_dir: str, http_prefix: str,
nickname: str, domain: str,
post_json_object: {}, translate: {},
yt_replace_domain: str,
twitter_replacement_domain: str,
allow_local_network_access: bool,
recent_posts_cache: {}, debug: bool,
system_language: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
bold_reading: bool) -> bool:
"""Returns true if the given post has attached image media
"""
if post_json_object['type'] == 'Announce':
blocked_cache = {}
post_json_announce = \
download_announce(session, base_dir, http_prefix,
nickname, domain, post_json_object,
__version__, translate,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
blocked_cache, bold_reading)
if post_json_announce:
post_json_object = post_json_announce
if post_json_object['type'] != 'Create':
return False
if not has_object_dict(post_json_object):
return False
if post_json_object['object'].get('moderationStatus'):
return False
if post_json_object['object']['type'] != 'Note' and \
post_json_object['object']['type'] != 'Page' and \
post_json_object['object']['type'] != 'Event' and \
post_json_object['object']['type'] != 'ChatMessage' and \
post_json_object['object']['type'] != 'Article':
return False
if not post_json_object['object'].get('attachment'):
return False
if not isinstance(post_json_object['object']['attachment'], list):
return False
for attach in post_json_object['object']['attachment']:
if attach.get('mediaType') and attach.get('url'):
if attach['mediaType'].startswith('image/') or \
attach['mediaType'].startswith('audio/') or \
attach['mediaType'].startswith('video/'):
return True
return False
def _add_post_string_to_timeline(post_str: str, boxname: str,
posts_in_box: [], box_actor: str) -> bool:
""" is this a valid timeline post?
"""
# must be a recognized ActivityPub type
if ('"Note"' in post_str or
'"EncryptedMessage"' in post_str or
'"ChatMessage"' in post_str or
'"Event"' in post_str or
'"Article"' in post_str or
'"Patch"' in post_str or
'"Announce"' in post_str or
('"Question"' in post_str and
('"Create"' in post_str or '"Update"' in post_str))):
if boxname == 'dm':
if '#Public' in post_str or '/followers' in post_str:
return False
elif boxname == 'tlreplies':
if box_actor not in post_str:
return False
elif boxname in ('tlblogs', 'tlnews', 'tlfeatures'):
if '"Create"' not in post_str:
return False
if '"Article"' not in post_str:
return False
elif boxname == 'tlmedia':
if '"Create"' in post_str:
if ('mediaType' not in post_str or
('image/' not in post_str and
'video/' not in post_str and
'audio/' not in post_str)):
return False
# add the post to the dictionary
posts_in_box.append(post_str)
return True
return False
def _add_post_to_timeline(file_path: str, boxname: str,
posts_in_box: [], box_actor: str) -> bool:
""" Reads a post from file and decides whether it is valid
"""
with open(file_path, 'r') as post_file:
post_str = post_file.read()
if file_path.endswith('.json'):
replies_filename = file_path.replace('.json', '.replies')
if os.path.isfile(replies_filename):
# append a replies identifier, which will later be removed
post_str += '<hasReplies>'
mitm_filename = file_path.replace('.json', '.mitm')
if os.path.isfile(mitm_filename):
# append a mitm identifier, which will later be removed
post_str += '<postmitm>'
return _add_post_string_to_timeline(post_str, boxname, posts_in_box,
box_actor)
return False
def remove_post_interactions(post_json_object: {}, force: bool) -> bool:
""" Don't show likes, replies, bookmarks, DMs or shares (announces) to
unauthorized viewers. This makes the timeline less useful to
marketers and other surveillance-oriented organizations.
Returns False if this is a private post
"""
has_object = False
if has_object_dict(post_json_object):
has_object = True
if has_object:
post_obj = post_json_object['object']
if not force:
# If not authorized and it's a private post
# then just don't show it within timelines
if not is_public_post(post_json_object):
return False
else:
post_obj = post_json_object
# clear the likes
if post_obj.get('likes'):
post_obj['likes'] = {
'items': []
}
# clear the reactions
if post_obj.get('reactions'):
post_obj['reactions'] = {
'items': []
}
# remove other collections
remove_collections = (
'replies', 'shares', 'bookmarks', 'ignores'
)
for remove_name in remove_collections:
if post_obj.get(remove_name):
post_obj[remove_name] = {}
return True
def _passed_newswire_voting(newswire_votes_threshold: int,
base_dir: str, domain: str,
post_filename: str,
positive_voting: bool,
voting_time_mins: int) -> bool:
"""Returns true if the post has passed through newswire voting
"""
# apply votes within this timeline
if newswire_votes_threshold <= 0:
return True
# note that the presence of an arrival file also indicates
# that this post is moderated
arrival_date = \
locate_news_arrival(base_dir, domain, post_filename)
if not arrival_date:
return True
# how long has elapsed since this post arrived?
curr_date = datetime.datetime.utcnow()
time_diff_mins = \
int((curr_date - arrival_date).total_seconds() / 60)
# has the voting time elapsed?
if time_diff_mins < voting_time_mins:
# voting is still happening, so don't add this
# post to the timeline
return False
# if there a votes file for this post?
votes_filename = \
locate_news_votes(base_dir, domain, post_filename)
if not votes_filename:
return True
# load the votes file and count the votes
votes_json = load_json(votes_filename, 0, 2)
if not votes_json:
return True
if not positive_voting:
if votes_on_newswire_item(votes_json) >= \
newswire_votes_threshold:
# Too many veto votes.
# Continue without incrementing
# the posts counter
return False
else:
if votes_on_newswire_item < \
newswire_votes_threshold:
# Not enough votes.
# Continue without incrementing
# the posts counter
return False
return True
def _create_box_indexed(recent_posts_cache: {},
session, base_dir: str, boxname: str,
nickname: str, domain: str, port: int,
http_prefix: str,
items_per_page: int, header_only: bool,
authorized: bool,
newswire_votes_threshold: int, positive_voting: bool,
voting_time_mins: int, page_number: int) -> {}:
"""Constructs the box feed for a person with the given nickname
"""
if not authorized or not page_number:
page_number = 1
if boxname not in ('inbox', 'dm', 'tlreplies', 'tlmedia',
'tlblogs', 'tlnews', 'tlfeatures', 'outbox',
'tlbookmarks', 'bookmarks'):
print('ERROR: invalid boxname ' + boxname)
return None
# bookmarks and events timelines are like the inbox
# but have their own separate index
index_box_name = boxname
timeline_nickname = nickname
if boxname == "tlbookmarks":
boxname = "bookmarks"
index_box_name = boxname
elif boxname == "tlfeatures":
boxname = "tlblogs"
index_box_name = boxname
timeline_nickname = 'news'
original_domain = domain
domain = get_full_domain(domain, port)
box_actor = local_actor_url(http_prefix, nickname, domain)
page_str = '?page=true'
if page_number:
if page_number < 1:
page_number = 1
try:
page_str = '?page=' + str(page_number)
except BaseException:
print('EX: _create_box_indexed ' +
'unable to convert page number to string')
box_url = local_actor_url(http_prefix, nickname, domain) + '/' + boxname
box_header = {
'@context': 'https://www.w3.org/ns/activitystreams',
'first': box_url + '?page=true',
'id': box_url,
'last': box_url + '?page=true',
'totalItems': 0,
'type': 'OrderedCollection'
}
box_items = {
'@context': 'https://www.w3.org/ns/activitystreams',
'id': box_url + page_str,
'orderedItems': [
],
'partOf': box_url,
'type': 'OrderedCollectionPage'
}
posts_in_box = []
post_urls_in_box = []
index_filename = \
acct_dir(base_dir, timeline_nickname, original_domain) + \
'/' + index_box_name + '.index'
total_posts_count = 0
posts_added_to_timeline = 0
if os.path.isfile(index_filename):
with open(index_filename, 'r') as index_file:
posts_added_to_timeline = 0
while posts_added_to_timeline < items_per_page:
post_filename = index_file.readline()
if not post_filename:
break
# Has this post passed through the newswire voting stage?
if not _passed_newswire_voting(newswire_votes_threshold,
base_dir, domain,
post_filename,
positive_voting,
voting_time_mins):
continue
# Skip through any posts previous to the current page
if total_posts_count < int((page_number - 1) * items_per_page):
total_posts_count += 1
continue
# if this is a full path then remove the directories
if '/' in post_filename:
post_filename = post_filename.split('/')[-1]
# filename of the post without any extension or path
# This should also correspond to any index entry in
# the posts cache
post_url = \
post_filename.replace('\n', '').replace('\r', '')
post_url = post_url.replace('.json', '').strip()
if post_url in post_urls_in_box:
continue
# is the post cached in memory?
if recent_posts_cache.get('index'):
if post_url in recent_posts_cache['index']:
if recent_posts_cache['json'].get(post_url):
url = recent_posts_cache['json'][post_url]
if _add_post_string_to_timeline(url,
boxname,
posts_in_box,
box_actor):
total_posts_count += 1
posts_added_to_timeline += 1
post_urls_in_box.append(post_url)
continue
print('Post not added to timeline')
# read the post from file
full_post_filename = \
locate_post(base_dir, nickname,
original_domain, post_url, False)
if full_post_filename:
# has the post been rejected?
if os.path.isfile(full_post_filename + '.reject'):
continue
if _add_post_to_timeline(full_post_filename, boxname,
posts_in_box, box_actor):
posts_added_to_timeline += 1
total_posts_count += 1
post_urls_in_box.append(post_url)
else:
print('WARN: Unable to add post ' + post_url +
' nickname ' + nickname +
' timeline ' + boxname)
else:
if timeline_nickname != nickname:
# if this is the features timeline
full_post_filename = \
locate_post(base_dir, timeline_nickname,
original_domain, post_url, False)
if full_post_filename:
if _add_post_to_timeline(full_post_filename,
boxname,
posts_in_box, box_actor):
posts_added_to_timeline += 1
total_posts_count += 1
post_urls_in_box.append(post_url)
else:
print('WARN: Unable to add features post ' +
post_url + ' nickname ' + nickname +
' timeline ' + boxname)
else:
print('WARN: features timeline. ' +
'Unable to locate post ' + post_url)
else:
if timeline_nickname == 'news':
print('WARN: Unable to locate news post ' +
post_url + ' nickname ' + nickname)
else:
print('WARN: Unable to locate post ' + post_url +
' nickname ' + nickname)
if total_posts_count < 3:
print('Posts added to json timeline ' + boxname + ': ' +
str(posts_added_to_timeline))
# Generate first and last entries within header
if total_posts_count > 0:
last_page = int(total_posts_count / items_per_page)
if last_page < 1:
last_page = 1
box_header['last'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + str(last_page)
if header_only:
box_header['totalItems'] = len(posts_in_box)
prev_page_str = 'true'
if page_number > 1:
prev_page_str = str(page_number - 1)
box_header['prev'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + prev_page_str
next_page_str = str(page_number + 1)
box_header['next'] = \
local_actor_url(http_prefix, nickname, domain) + \
'/' + boxname + '?page=' + next_page_str
return box_header
for post_str in posts_in_box:
# Check if the post has replies
has_replies = False
if post_str.endswith('<hasReplies>'):
has_replies = True
# remove the replies identifier
post_str = post_str.replace('<hasReplies>', '')
# Check if the post was delivered via a third party
mitm = False
if post_str.endswith('<postmitm>'):
mitm = True
# remove the mitm identifier
post_str = post_str.replace('<postmitm>', '')
pst = None
try:
pst = json.loads(post_str)
except BaseException:
print('EX: _create_box_indexed unable to load json ' + post_str)
continue
# Does this post have replies?
# This will be used to indicate that replies exist within the html
# created by individual_post_as_html
pst['hasReplies'] = has_replies
# was the post delivered via a third party?
pst['mitm'] = mitm
if not authorized:
if not remove_post_interactions(pst, False):
continue
box_items['orderedItems'].append(pst)
return box_items
def expire_cache(base_dir: str, person_cache: {},
http_prefix: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box: int):
"""Thread used to expire actors from the cache and archive old posts
"""
while True:
# once per day
time.sleep(60 * 60 * 24)
expire_person_cache(person_cache)
archive_posts(base_dir, http_prefix, archive_dir, recent_posts_cache,
max_posts_in_box)
def archive_posts(base_dir: str, http_prefix: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box=32000) -> None:
"""Archives posts for all accounts
"""
if max_posts_in_box == 0:
return
if archive_dir:
if not os.path.isdir(archive_dir):
os.mkdir(archive_dir)
if archive_dir:
if not os.path.isdir(archive_dir + '/accounts'):
os.mkdir(archive_dir + '/accounts')
for _, dirs, _ in os.walk(base_dir + '/accounts'):
for handle in dirs:
if '@' in handle:
nickname = handle.split('@')[0]
domain = handle.split('@')[1]
archive_subdir = None
if archive_dir:
if not os.path.isdir(archive_dir + '/accounts/' + handle):
os.mkdir(archive_dir + '/accounts/' + handle)
if not os.path.isdir(archive_dir + '/accounts/' +
handle + '/inbox'):
os.mkdir(archive_dir + '/accounts/' +
handle + '/inbox')
if not os.path.isdir(archive_dir + '/accounts/' +
handle + '/outbox'):
os.mkdir(archive_dir + '/accounts/' +
handle + '/outbox')
archive_subdir = archive_dir + '/accounts/' + \
handle + '/inbox'
archive_posts_for_person(http_prefix,
nickname, domain, base_dir,
'inbox', archive_subdir,
recent_posts_cache, max_posts_in_box)
if archive_dir:
archive_subdir = archive_dir + '/accounts/' + \
handle + '/outbox'
archive_posts_for_person(http_prefix,
nickname, domain, base_dir,
'outbox', archive_subdir,
recent_posts_cache, max_posts_in_box)
break
def archive_posts_for_person(http_prefix: str, nickname: str, domain: str,
base_dir: str,
boxname: str, archive_dir: str,
recent_posts_cache: {},
max_posts_in_box=32000) -> None:
"""Retain a maximum number of posts within the given box
Move any others to an archive directory
"""
if boxname not in ('inbox', 'outbox'):
return
if archive_dir:
if not os.path.isdir(archive_dir):
os.mkdir(archive_dir)
box_dir = create_person_dir(nickname, domain, base_dir, boxname)
posts_in_box = os.scandir(box_dir)
no_of_posts = 0
for _ in posts_in_box:
no_of_posts += 1
if no_of_posts <= max_posts_in_box:
print('Checked ' + str(no_of_posts) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
return
# remove entries from the index
handle = nickname + '@' + domain
index_filename = \
base_dir + '/accounts/' + handle + '/' + boxname + '.index'
if os.path.isfile(index_filename):
index_ctr = 0
# get the existing index entries as a string
new_index = ''
with open(index_filename, 'r') as index_file:
for post_id in index_file:
new_index += post_id
index_ctr += 1
if index_ctr >= max_posts_in_box:
break
# save the new index file
if len(new_index) > 0:
with open(index_filename, 'w+') as index_file:
index_file.write(new_index)
posts_in_box_dict = {}
posts_ctr = 0
posts_in_box = os.scandir(box_dir)
for post_filename in posts_in_box:
post_filename = post_filename.name
if not post_filename.endswith('.json'):
continue
# Time of file creation
full_filename = os.path.join(box_dir, post_filename)
if os.path.isfile(full_filename):
content = open(full_filename).read()
if '"published":' in content:
published_str = content.split('"published":')[1]
if '"' in published_str:
published_str = published_str.split('"')[1]
if published_str.endswith('Z'):
posts_in_box_dict[published_str] = post_filename
posts_ctr += 1
no_of_posts = posts_ctr
if no_of_posts <= max_posts_in_box:
print('Checked ' + str(no_of_posts) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
return
# sort the list in ascending order of date
posts_in_box_sorted = \
OrderedDict(sorted(posts_in_box_dict.items(), reverse=False))
# directory containing cached html posts
post_cache_dir = box_dir.replace('/' + boxname, '/postcache')
remove_ctr = 0
for published_str, post_filename in posts_in_box_sorted.items():
file_path = os.path.join(box_dir, post_filename)
if not os.path.isfile(file_path):
continue
if archive_dir:
archive_path = os.path.join(archive_dir, post_filename)
os.rename(file_path, archive_path)
extensions = ('replies', 'votes', 'arrived', 'muted')
for ext in extensions:
ext_path = file_path.replace('.json', '.' + ext)
if os.path.isfile(ext_path):
os.rename(ext_path,
archive_path.replace('.json', '.' + ext))
else:
ext_path = file_path.replace('.json',
'.json.' + ext)
if os.path.isfile(ext_path):
os.rename(ext_path,
archive_path.replace('.json',
'.json.' + ext))
else:
delete_post(base_dir, http_prefix, nickname, domain,
file_path, False, recent_posts_cache, False)
# remove cached html posts
post_cache_filename = \
os.path.join(post_cache_dir, post_filename)
post_cache_filename = post_cache_filename.replace('.json', '.html')
if os.path.isfile(post_cache_filename):
try:
os.remove(post_cache_filename)
except OSError:
print('EX: archive_posts_for_person unable to delete ' +
post_cache_filename)
no_of_posts -= 1
remove_ctr += 1
if no_of_posts <= max_posts_in_box:
break
if archive_dir:
print('Archived ' + str(remove_ctr) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
else:
print('Removed ' + str(remove_ctr) + ' ' + boxname +
' posts for ' + nickname + '@' + domain)
print(nickname + '@' + domain + ' has ' + str(no_of_posts) +
' in ' + boxname)
def get_public_posts_of_person(base_dir: str, nickname: str, domain: str,
raw: bool, simple: bool, proxy_type: str,
port: int, http_prefix: str,
debug: bool, project_version: str,
system_language: str,
signing_priv_key_pem: str,
origin_domain: str) -> None:
""" This is really just for test purposes
"""
if debug:
if signing_priv_key_pem:
print('Signing key available')
else:
print('Signing key missing')
print('Starting new session for getting public posts')
session = create_session(proxy_type)
if not session:
if debug:
print('Session was not created')
return
person_cache = {}
cached_webfingers = {}
federation_list = []
group_account = False
if nickname.startswith('!'):
nickname = nickname[1:]
group_account = True
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
origin_domain, project_version, debug, group_account,
signing_priv_key_pem)
if not wf_request:
if debug:
print('No webfinger result was returned for ' + handle)
sys.exit()
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
sys.exit()
if debug:
print('Getting the outbox for ' + handle)
(person_url, _, _, person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
62524)
if debug:
print('Actor url: ' + str(person_id))
if not person_id:
return
max_mentions = 10
max_emoji = 10
max_attachments = 5
_get_posts(session, person_url, 30, max_mentions, max_emoji,
max_attachments, federation_list,
person_cache, raw, simple, debug,
project_version, http_prefix, origin_domain, system_language,
signing_priv_key_pem)
def get_public_post_domains(session, base_dir: str, nickname: str, domain: str,
origin_domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, domain_list: [],
system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a list of domains referenced within public posts
"""
if not session:
session = create_session(proxy_type)
if not session:
return domain_list
person_cache = {}
cached_webfingers = {}
federation_list = []
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return domain_list
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return domain_list
(person_url, _, _, _, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
92522)
max_mentions = 99
max_emoji = 99
max_attachments = 5
post_domains = \
get_post_domains(session, person_url, 64, max_mentions, max_emoji,
max_attachments, federation_list,
person_cache, debug,
project_version, http_prefix, domain,
word_frequency, domain_list, system_language,
signing_priv_key_pem)
post_domains.sort()
return post_domains
def download_follow_collection(signing_priv_key_pem: str,
follow_type: str,
session, http_prefix: str,
actor: str, page_number: int,
no_of_pages: int, debug: bool) -> []:
"""Returns a list of following/followers for the given actor
by downloading the json for their following/followers collection
"""
prof = 'https://www.w3.org/ns/activitystreams'
if '/channel/' not in actor or '/accounts/' not in actor:
accept_str = \
'application/activity+json; ' + \
'profile="' + prof + '"'
session_headers = {
'Accept': accept_str
}
else:
accept_str = \
'application/ld+json; ' + \
'profile="' + prof + '"'
session_headers = {
'Accept': accept_str
}
result = []
for page_ctr in range(no_of_pages):
url = \
actor + '/' + follow_type + '?page=' + str(page_number + page_ctr)
followers_json = \
get_json(signing_priv_key_pem, session, url, session_headers, None,
debug, __version__, http_prefix, None)
if followers_json:
if followers_json.get('orderedItems'):
for follower_actor in followers_json['orderedItems']:
if follower_actor not in result:
result.append(follower_actor)
elif followers_json.get('items'):
for follower_actor in followers_json['items']:
if follower_actor not in result:
result.append(follower_actor)
else:
break
else:
break
return result
def get_public_post_info(session, base_dir: str, nickname: str, domain: str,
origin_domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a dict of domains referenced within public posts
"""
if not session:
session = create_session(proxy_type)
if not session:
return {}
person_cache = {}
cached_webfingers = {}
federation_list = []
domain_full = get_full_domain(domain, port)
handle = http_prefix + "://" + domain_full + "/@" + nickname
wf_request = \
webfinger_handle(session, handle, http_prefix, cached_webfingers,
domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
return {}
if not isinstance(wf_request, dict):
print('Webfinger for ' + handle + ' did not return a dict. ' +
str(wf_request))
return {}
(person_url, _, _, _, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
nickname, domain, 'outbox',
13863)
max_mentions = 99
max_emoji = 99
max_attachments = 5
max_posts = 64
post_domains = \
get_post_domains(session, person_url, max_posts,
max_mentions, max_emoji,
max_attachments, federation_list,
person_cache, debug,
project_version, http_prefix, domain,
word_frequency, [], system_language,
signing_priv_key_pem)
post_domains.sort()
domains_info = {}
for pdomain in post_domains:
if not domains_info.get(pdomain):
domains_info[pdomain] = []
blocked_posts = \
_get_posts_for_blocked_domains(base_dir, session,
person_url, max_posts,
max_mentions,
max_emoji, max_attachments,
federation_list,
person_cache,
debug,
project_version, http_prefix,
domain, signing_priv_key_pem)
for blocked_domain, post_url_list in blocked_posts.items():
domains_info[blocked_domain] += post_url_list
return domains_info
def get_public_post_domains_blocked(session, base_dir: str,
nickname: str, domain: str,
proxy_type: str, port: int,
http_prefix: str,
debug: bool, project_version: str,
word_frequency: {}, domain_list: [],
system_language: str,
signing_priv_key_pem: str) -> []:
""" Returns a list of domains referenced within public posts which
are globally blocked on this instance
"""
origin_domain = domain
post_domains = \
get_public_post_domains(session, base_dir, nickname, domain,
origin_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, domain_list, system_language,
signing_priv_key_pem)
if not post_domains:
return []
blocking_filename = base_dir + '/accounts/blocking.txt'
if not os.path.isfile(blocking_filename):
return []
# read the blocked domains as a single string
blocked_str = ''
with open(blocking_filename, 'r') as fp_block:
blocked_str = fp_block.read()
blocked_domains = []
for domain_name in post_domains:
if '@' not in domain_name:
continue
# get the domain after the @
domain_name = domain_name.split('@')[1].strip()
if is_evil(domain_name):
blocked_domains.append(domain_name)
continue
if domain_name in blocked_str:
blocked_domains.append(domain_name)
return blocked_domains
def _get_non_mutuals_of_person(base_dir: str,
nickname: str, domain: str) -> []:
"""Returns the followers who are not mutuals of a person
i.e. accounts which follow you but you don't follow them
"""
followers = \
get_followers_list(base_dir, nickname, domain, 'followers.txt')
following = \
get_followers_list(base_dir, nickname, domain, 'following.txt')
non_mutuals = []
for handle in followers:
if handle not in following:
non_mutuals.append(handle)
return non_mutuals
def check_domains(session, base_dir: str,
nickname: str, domain: str,
proxy_type: str, port: int, http_prefix: str,
debug: bool, project_version: str,
max_blocked_domains: int, single_check: bool,
system_language: str,
signing_priv_key_pem: str) -> None:
"""Checks follower accounts for references to globally blocked domains
"""
word_frequency = {}
non_mutuals = _get_non_mutuals_of_person(base_dir, nickname, domain)
if not non_mutuals:
print('No non-mutual followers were found')
return
follower_warning_filename = base_dir + '/accounts/followerWarnings.txt'
update_follower_warnings = False
follower_warning_str = ''
if os.path.isfile(follower_warning_filename):
with open(follower_warning_filename, 'r') as fp_warn:
follower_warning_str = fp_warn.read()
if single_check:
# checks a single random non-mutual
index = random.randrange(0, len(non_mutuals))
handle = non_mutuals[index]
if '@' in handle:
non_mutual_nickname = handle.split('@')[0]
non_mutual_domain = handle.split('@')[1].strip()
blocked_domains = \
get_public_post_domains_blocked(session, base_dir,
non_mutual_nickname,
non_mutual_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, [],
system_language,
signing_priv_key_pem)
if blocked_domains:
if len(blocked_domains) > max_blocked_domains:
follower_warning_str += handle + '\n'
update_follower_warnings = True
else:
# checks all non-mutuals
for handle in non_mutuals:
if '@' not in handle:
continue
if handle in follower_warning_str:
continue
non_mutual_nickname = handle.split('@')[0]
non_mutual_domain = handle.split('@')[1].strip()
blocked_domains = \
get_public_post_domains_blocked(session, base_dir,
non_mutual_nickname,
non_mutual_domain,
proxy_type, port, http_prefix,
debug, project_version,
word_frequency, [],
system_language,
signing_priv_key_pem)
if blocked_domains:
print(handle)
for bdomain in blocked_domains:
print(' ' + bdomain)
if len(blocked_domains) > max_blocked_domains:
follower_warning_str += handle + '\n'
update_follower_warnings = True
if update_follower_warnings and follower_warning_str:
with open(follower_warning_filename, 'w+') as fp_warn:
fp_warn.write(follower_warning_str)
if not single_check:
print(follower_warning_str)
def populate_replies_json(base_dir: str, nickname: str, domain: str,
post_replies_filename: str, authorized: bool,
replies_json: {}) -> None:
pub_str = 'https://www.w3.org/ns/activitystreams#Public'
# populate the items list with replies
replies_boxes = ('outbox', 'inbox')
with open(post_replies_filename, 'r') as replies_file:
for message_id in replies_file:
reply_found = False
# examine inbox and outbox
for boxname in replies_boxes:
message_id2 = message_id.replace('\n', '').replace('\r', '')
search_filename = \
acct_dir(base_dir, nickname, domain) + '/' + \
boxname + '/' + \
message_id2.replace('/', '#') + '.json'
if os.path.isfile(search_filename):
if authorized or \
pub_str in open(search_filename).read():
post_json_object = load_json(search_filename)
if post_json_object:
if post_json_object['object'].get('cc'):
pjo = post_json_object
if (authorized or
(pub_str in pjo['object']['to'] or
pub_str in pjo['object']['cc'])):
replies_json['orderedItems'].append(pjo)
reply_found = True
else:
if authorized or \
pub_str in post_json_object['object']['to']:
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
reply_found = True
break
# if not in either inbox or outbox then examine the shared inbox
if not reply_found:
message_id2 = message_id.replace('\n', '').replace('\r', '')
search_filename = \
base_dir + \
'/accounts/inbox@' + \
domain + '/inbox/' + \
message_id2.replace('/', '#') + '.json'
if os.path.isfile(search_filename):
if authorized or \
pub_str in open(search_filename).read():
# get the json of the reply and append it to
# the collection
post_json_object = load_json(search_filename)
if post_json_object:
if post_json_object['object'].get('cc'):
pjo = post_json_object
if (authorized or
(pub_str in pjo['object']['to'] or
pub_str in pjo['object']['cc'])):
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
else:
if authorized or \
pub_str in post_json_object['object']['to']:
pjo = post_json_object
replies_json['orderedItems'].append(pjo)
def _reject_announce(announce_filename: str,
base_dir: str, nickname: str, domain: str,
announce_post_id: str, recent_posts_cache: {}):
"""Marks an announce as rejected
"""
reject_post_id(base_dir, nickname, domain, announce_post_id,
recent_posts_cache)
# reject the post referenced by the announce activity object
if not os.path.isfile(announce_filename + '.reject'):
with open(announce_filename + '.reject', 'w+') as reject_announce_file:
reject_announce_file.write('\n')
def download_announce(session, base_dir: str, http_prefix: str,
nickname: str, domain: str,
post_json_object: {}, project_version: str,
translate: {},
yt_replace_domain: str,
twitter_replacement_domain: str,
allow_local_network_access: bool,
recent_posts_cache: {}, debug: bool,
system_language: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
blocked_cache: {}, bold_reading: bool) -> {}:
"""Download the post referenced by an announce
"""
if not post_json_object.get('object'):
return None
if not isinstance(post_json_object['object'], str):
return None
# ignore self-boosts
if post_json_object['actor'] in post_json_object['object']:
return None
# get the announced post
announce_cache_dir = base_dir + '/cache/announce/' + nickname
if not os.path.isdir(announce_cache_dir):
os.mkdir(announce_cache_dir)
post_id = None
if post_json_object.get('id'):
post_id = remove_id_ending(post_json_object['id'])
announce_filename = \
announce_cache_dir + '/' + \
post_json_object['object'].replace('/', '#') + '.json'
if os.path.isfile(announce_filename + '.reject'):
return None
if os.path.isfile(announce_filename):
if debug:
print('Reading cached Announce content for ' +
post_json_object['object'])
post_json_object = load_json(announce_filename)
if post_json_object:
return post_json_object
else:
profile_str = 'https://www.w3.org/ns/activitystreams'
accept_str = \
'application/activity+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
if '/channel/' in post_json_object['actor'] or \
'/accounts/' in post_json_object['actor']:
accept_str = \
'application/ld+json; ' + \
'profile="' + profile_str + '"'
as_header = {
'Accept': accept_str
}
actor_nickname = get_nickname_from_actor(post_json_object['actor'])
if not actor_nickname:
print('WARN: download_announce no actor_nickname')
return None
actor_domain, actor_port = \
get_domain_from_actor(post_json_object['actor'])
if not actor_domain:
print('Announce actor does not contain a ' +
'valid domain or port number: ' +
str(post_json_object['actor']))
return None
if is_blocked(base_dir, nickname, domain,
actor_nickname, actor_domain):
print('Announce download blocked actor: ' +
actor_nickname + '@' + actor_domain)
return None
object_nickname = get_nickname_from_actor(post_json_object['object'])
object_domain, _ = \
get_domain_from_actor(post_json_object['object'])
if not object_domain:
print('Announce object does not contain a ' +
'valid domain or port number: ' +
str(post_json_object['object']))
return None
if is_blocked(base_dir, nickname, domain, object_nickname,
object_domain):
if object_nickname and object_domain:
print('Announce download blocked object: ' +
object_nickname + '@' + object_domain)
else:
print('Announce download blocked object: ' +
str(post_json_object['object']))
return None
if debug:
print('Downloading Announce content for ' +
post_json_object['object'])
announced_json = \
get_json(signing_priv_key_pem, session,
post_json_object['object'],
as_header, None, debug, project_version,
http_prefix, domain)
if not announced_json:
return None
if not isinstance(announced_json, dict):
print('WARN: announced post json is not a dict - ' +
post_json_object['object'])
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not announced_json.get('id'):
print('WARN: announced post does not have an id ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not announced_json.get('type'):
print('WARN: announced post does not have a type ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json['type'] == 'Video':
converted_json = \
convert_video_to_note(base_dir, nickname, domain,
system_language,
announced_json, blocked_cache)
if converted_json:
announced_json = converted_json
if '/statuses/' not in announced_json['id']:
print('WARN: announced post id does not contain /statuses/ ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not has_users_path(announced_json['id']):
print('WARN: announced post id does not contain /users/ ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if announced_json['type'] != 'Note' and \
announced_json['type'] != 'Page' and \
announced_json['type'] != 'Article':
print('WARN: announced post is not Note/Page/Article ' +
str(announced_json))
# You can only announce Note or Article types
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not announced_json.get('content'):
print('WARN: announced post does not have content ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not announced_json.get('published'):
print('WARN: announced post does not have published ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if '.' in announced_json['published'] and \
'Z' in announced_json['published']:
announced_json['published'] = \
announced_json['published'].split('.')[0] + 'Z'
if not valid_post_date(announced_json['published'], 90, debug):
print('WARN: announced post is not recently published ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if not understood_post_language(base_dir, nickname, domain,
announced_json, system_language,
http_prefix, domain_full,
person_cache):
return None
# Check the content of the announce
content_str = announced_json['content']
using_content_map = False
if announced_json.get('contentMap'):
if announced_json['contentMap'].get(system_language):
content_str = announced_json['contentMap'][system_language]
using_content_map = True
if dangerous_markup(content_str, allow_local_network_access):
print('WARN: announced post contains dangerous markup ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if is_filtered(base_dir, nickname, domain, content_str):
print('WARN: announced post has been filtered ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
if invalid_ciphertext(content_str):
print('WARN: announced post contains invalid ciphertext ' +
str(announced_json))
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
print('WARN: Invalid ciphertext within announce ' +
str(announced_json))
return None
# remove any long words
content_str = remove_long_words(content_str, 40, [])
# Prevent the same word from being repeated many times
content_str = limit_repeated_words(content_str, 6)
# remove text formatting, such as bold/italics
content_str = remove_text_formatting(content_str, bold_reading)
# set the content after santitization
if using_content_map:
announced_json['contentMap'][system_language] = content_str
announced_json['content'] = content_str
# wrap in create to be consistent with other posts
announced_json = \
outbox_message_create_wrap(http_prefix,
actor_nickname, actor_domain,
actor_port, announced_json)
if announced_json['type'] != 'Create':
print('WARN: announced post could not be wrapped in Create ' +
str(announced_json))
# Create wrap failed
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
# labelAccusatoryPost(post_json_object, translate)
# set the id to the original status
announced_json['id'] = post_json_object['object']
announced_json['object']['id'] = post_json_object['object']
# check that the repeat isn't for a blocked account
attributed_nickname = \
get_nickname_from_actor(announced_json['object']['id'])
attributed_domain, attributed_port = \
get_domain_from_actor(announced_json['object']['id'])
if attributed_nickname and attributed_domain:
attributed_domain = \
get_full_domain(attributed_domain, attributed_port)
if is_blocked(base_dir, nickname, domain,
attributed_nickname, attributed_domain):
print('WARN: announced post handle is blocked ' +
str(attributed_nickname) + '@' + attributed_domain)
_reject_announce(announce_filename,
base_dir, nickname, domain, post_id,
recent_posts_cache)
return None
post_json_object = announced_json
replace_you_tube(post_json_object, yt_replace_domain, system_language)
replace_twitter(post_json_object, twitter_replacement_domain,
system_language)
if save_json(post_json_object, announce_filename):
return post_json_object
return None
def is_muted_conv(base_dir: str, nickname: str, domain: str, post_id: str,
conversation_id: str) -> bool:
"""Returns true if the given post is muted
"""
if conversation_id:
conv_muted_filename = \
acct_dir(base_dir, nickname, domain) + '/conversation/' + \
conversation_id.replace('/', '#') + '.muted'
if os.path.isfile(conv_muted_filename):
return True
post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename:
return False
if os.path.isfile(post_filename + '.muted'):
return True
return False
def send_block_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, blocked_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str) -> {}:
"""Creates a block via c2s
"""
if not session:
print('WARN: No session for send_block_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
to_url = 'https://www.w3.org/ns/activitystreams#Public'
cc_url = block_actor + '/followers'
new_block_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Block',
'actor': block_actor,
'object': blocked_url,
'to': [to_url],
'cc': [cc_url]
}
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: block webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: block Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652)
if not inbox_url:
if debug:
print('DEBUG: block no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: block no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_block_json, [], inbox_url,
headers, 30, True)
if not post_result:
print('WARN: block unable to post')
if debug:
print('DEBUG: c2s POST block success')
return new_block_json
def send_mute_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, muted_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str) -> {}:
"""Creates a mute via c2s
"""
if not session:
print('WARN: No session for send_mute_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
handle = replace_users_with_at(actor)
new_mute_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Ignore',
'actor': actor,
'to': [actor],
'object': muted_url
}
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: mute webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: mute Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652)
if not inbox_url:
if debug:
print('DEBUG: mute no ' + post_to_box + ' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: mute no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_mute_json, [], inbox_url,
headers, 3, True)
if post_result is None:
print('WARN: mute unable to post')
if debug:
print('DEBUG: c2s POST mute success')
return new_mute_json
def send_undo_mute_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, muted_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str) -> {}:
"""Undoes a mute via c2s
"""
if not session:
print('WARN: No session for send_undo_mute_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
handle = replace_users_with_at(actor)
undo_mute_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Undo',
'actor': actor,
'to': [actor],
'object': {
'type': 'Ignore',
'actor': actor,
'to': [actor],
'object': muted_url
}
}
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: undo mute webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: undo mute Webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 72652)
if not inbox_url:
if debug:
print('DEBUG: undo mute no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: undo mute no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, undo_mute_json, [], inbox_url,
headers, 3, True)
if post_result is None:
print('WARN: undo mute unable to post')
if debug:
print('DEBUG: c2s POST undo mute success')
return undo_mute_json
def send_undo_block_via_server(base_dir: str, session,
from_nickname: str, password: str,
from_domain: str, from_port: int,
http_prefix: str, blocked_url: str,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str) -> {}:
"""Creates a block via c2s
"""
if not session:
print('WARN: No session for send_block_via_server')
return 6
from_domain_full = get_full_domain(from_domain, from_port)
block_actor = local_actor_url(http_prefix, from_nickname, from_domain_full)
to_url = 'https://www.w3.org/ns/activitystreams#Public'
cc_url = block_actor + '/followers'
new_block_json = {
"@context": "https://www.w3.org/ns/activitystreams",
'type': 'Undo',
'actor': block_actor,
'object': {
'type': 'Block',
'actor': block_actor,
'object': blocked_url,
'to': [to_url],
'cc': [cc_url]
}
}
handle = http_prefix + '://' + from_domain_full + '/@' + from_nickname
# lookup the inbox for the To handle
wf_request = webfinger_handle(session, handle, http_prefix,
cached_webfingers,
from_domain, project_version, debug, False,
signing_priv_key_pem)
if not wf_request:
if debug:
print('DEBUG: unblock webfinger failed for ' + handle)
return 1
if not isinstance(wf_request, dict):
print('WARN: unblock webfinger for ' + handle +
' did not return a dict. ' + str(wf_request))
return 1
post_to_box = 'outbox'
# get the actor inbox for the To handle
origin_domain = from_domain
(inbox_url, _, _, from_person_id, _, _,
_, _) = get_person_box(signing_priv_key_pem,
origin_domain,
base_dir, session, wf_request,
person_cache,
project_version, http_prefix,
from_nickname,
from_domain, post_to_box, 53892)
if not inbox_url:
if debug:
print('DEBUG: unblock no ' + post_to_box +
' was found for ' + handle)
return 3
if not from_person_id:
if debug:
print('DEBUG: unblock no actor was found for ' + handle)
return 4
auth_header = create_basic_auth_header(from_nickname, password)
headers = {
'host': from_domain,
'Content-type': 'application/json',
'Authorization': auth_header
}
post_result = post_json(http_prefix, from_domain_full,
session, new_block_json, [], inbox_url,
headers, 30, True)
if not post_result:
print('WARN: unblock unable to post')
if debug:
print('DEBUG: c2s POST unblock success')
return new_block_json
def post_is_muted(base_dir: str, nickname: str, domain: str,
post_json_object: {}, message_id: str) -> bool:
""" Returns true if the given post is muted
"""
is_muted = None
if 'muted' in post_json_object:
is_muted = post_json_object['muted']
if is_muted is True or is_muted is False:
return is_muted
is_muted = False
post_dir = acct_dir(base_dir, nickname, domain)
mute_filename = \
post_dir + '/inbox/' + message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
else:
mute_filename = \
post_dir + '/outbox/' + \
message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
else:
mute_filename = \
base_dir + '/accounts/cache/announce/' + nickname + \
'/' + message_id.replace('/', '#') + '.json.muted'
if os.path.isfile(mute_filename):
is_muted = True
return is_muted
def c2s_box_json(base_dir: str, session,
nickname: str, password: str,
domain: str, port: int,
http_prefix: str,
box_name: str, page_number: int,
debug: bool, signing_priv_key_pem: str) -> {}:
"""C2S Authenticated GET of posts for a timeline
"""
if not session:
print('WARN: No session for c2s_box_json')
return None
domain_full = get_full_domain(domain, port)
actor = local_actor_url(http_prefix, nickname, domain_full)
auth_header = create_basic_auth_header(nickname, password)
profile_str = 'https://www.w3.org/ns/activitystreams'
headers = {
'host': domain,
'Content-type': 'application/json',
'Authorization': auth_header,
'Accept': 'application/ld+json; profile="' + profile_str + '"'
}
# GET json
url = actor + '/' + box_name + '?page=' + str(page_number)
box_json = get_json(signing_priv_key_pem, session, url, headers, None,
debug, __version__, http_prefix, None)
if box_json is not None and debug:
print('DEBUG: GET c2s_box_json success')
return box_json
def seconds_between_published(published1: str, published2: str) -> int:
"""Returns the number of seconds between two published dates
"""
try:
published1_time = \
datetime.datetime.strptime(published1, '%Y-%m-%dT%H:%M:%SZ')
except BaseException:
print('EX: seconds_between_published unable to parse date 1 ' +
str(published1))
return -1
try:
published2_time = \
datetime.datetime.strptime(published2, '%Y-%m-%dT%H:%M:%SZ')
except BaseException:
print('EX: seconds_between_published unable to parse date 2 ' +
str(published2))
return -1
return (published2_time - published1_time).seconds
def edited_post_filename(base_dir: str, nickname: str, domain: str,
post_json_object: {}, debug: bool,
max_time_diff_seconds: int,
system_language: str) -> (str, {}):
"""Returns the filename of the edited post
"""
if not has_object_dict(post_json_object):
return '', None
if not post_json_object.get('type'):
return '', None
if not post_json_object['object'].get('type'):
return '', None
if not post_json_object['object'].get('published'):
return '', None
if not post_json_object['object'].get('id'):
return '', None
if not post_json_object['object'].get('content'):
return '', None
if not post_json_object['object'].get('attributedTo'):
return '', None
if not isinstance(post_json_object['object']['attributedTo'], str):
return '', None
actor = post_json_object['object']['attributedTo']
actor_filename = \
acct_dir(base_dir, nickname, domain) + '/lastpost/' + \
actor.replace('/', '#')
if not os.path.isfile(actor_filename):
return '', None
post_id = remove_id_ending(post_json_object['object']['id'])
lastpost_id = None
try:
with open(actor_filename, 'r') as fp_actor:
lastpost_id = fp_actor.read()
except OSError:
print('EX: edited_post_filename unable to read ' + actor_filename)
return '', None
if not lastpost_id:
return '', None
if lastpost_id == post_id:
return '', None
lastpost_filename = \
locate_post(base_dir, nickname, domain, lastpost_id, False)
if not lastpost_filename:
return '', None
lastpost_json = load_json(lastpost_filename, 0)
if not lastpost_json:
return '', None
if not lastpost_json.get('type'):
return '', None
if lastpost_json['type'] != post_json_object['type']:
return '', None
if not lastpost_json['object'].get('type'):
return '', None
if lastpost_json['object']['type'] != post_json_object['object']['type']:
return '', None
if not lastpost_json['object'].get('published'):
return '', None
if not lastpost_json['object'].get('id'):
return '', None
if not lastpost_json['object'].get('content'):
return '', None
if not lastpost_json['object'].get('attributedTo'):
return '', None
if not isinstance(lastpost_json['object']['attributedTo'], str):
return '', None
time_diff_seconds = \
seconds_between_published(lastpost_json['object']['published'],
post_json_object['object']['published'])
if time_diff_seconds > max_time_diff_seconds:
return '', None
if debug:
print(post_id + ' might be an edit of ' + lastpost_id)
lastpost_content = lastpost_json['object']['content']
if lastpost_json['object'].get('contentMap'):
if lastpost_json['object']['contentMap'].get(system_language):
lastpost_content = \
lastpost_json['object']['contentMap'][system_language]
content = post_json_object['object']['content']
if post_json_object['object'].get('contentMap'):
if post_json_object['object']['contentMap'].get(system_language):
content = \
post_json_object['object']['contentMap'][system_language]
if words_similarity(lastpost_content, content, 10) < 70:
return '', None
print(post_id + ' is an edit of ' + lastpost_id)
return lastpost_filename, lastpost_json
def get_original_post_from_announce_url(announce_url: str, base_dir: str,
nickname: str,
domain: str) -> (str, str, str):
"""From the url of an announce this returns the actor, url and
filename (if available) of the original post being announced
"""
post_filename = locate_post(base_dir, nickname, domain, announce_url)
if not post_filename:
return None, None, None
announce_post_json = load_json(post_filename, 0, 1)
if not announce_post_json:
return None, None, post_filename
if not announce_post_json.get('type'):
return None, None, post_filename
if announce_post_json['type'] != 'Announce':
return None, None, post_filename
if not announce_post_json.get('object'):
return None, None, post_filename
if not isinstance(announce_post_json['object'], str):
return None, None, post_filename
actor = url = None
# do we have the original post?
orig_post_id = announce_post_json['object']
orig_filename = locate_post(base_dir, nickname, domain, orig_post_id)
if orig_filename:
# we have the original post
orig_post_json = load_json(orig_filename, 0, 1)
if orig_post_json:
if has_object_dict(orig_post_json):
if orig_post_json['object'].get('attributedTo'):
attrib = orig_post_json['object']['attributedTo']
if isinstance(attrib, str):
actor = orig_post_json['object']['attributedTo']
url = orig_post_id
elif orig_post_json['object'].get('actor'):
actor = orig_post_json['actor']
url = orig_post_id
else:
# we don't have the original post
if has_users_path(orig_post_id):
# get the actor from the original post url
orig_nick = get_nickname_from_actor(orig_post_id)
orig_domain, _ = get_domain_from_actor(orig_post_id)
if orig_nick and orig_domain:
actor = \
orig_post_id.split('/' + orig_nick + '/')[0] + \
'/' + orig_nick
url = orig_post_id
return actor, url, orig_filename