Merge branch 'main' of gitlab.com:bashrc2/epicyon

merge-requests/30/head
Bob Mottram 2022-11-17 20:27:35 +00:00
commit fb38442ed3
3 changed files with 441 additions and 237 deletions

339
daemon.py
View File

@ -114,6 +114,7 @@ from inbox import run_inbox_queue
from inbox import run_inbox_queue_watchdog
from inbox import save_post_to_inbox_queue
from inbox import populate_replies
from inbox import receive_edit_to_post
from follow import follower_approval_active
from follow import is_following_actor
from follow import get_following_feed
@ -439,6 +440,93 @@ SHARES_PER_PAGE = 12
class PubServer(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def _update_edited_post(self, base_dir: str,
nickname: str, domain: str,
message_json: {},
edited_published: str,
edited_postid: str,
recent_posts_cache: {},
box_name: str,
max_mentions: int, max_emoji: int,
allow_local_network_access: bool,
debug: bool,
system_language: str, http_prefix: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
max_recent_posts: int, translate: {},
session, cached_webfingers: {}, port: int,
allow_deletion: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
peertube_instances: [],
theme_name: str, max_like_count: int,
cw_lists: {}, dogwhistles: {},
min_images_for_accounts: []) -> None:
"""When an edited post is created this assigns
a published and updated date to it, and uses
the previous id
"""
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
message_json2 = message_json.copy()
receive_edit_to_post(recent_posts_cache,
message_json2,
base_dir,
nickname, domain,
max_mentions, max_emoji,
allow_local_network_access,
debug,
system_language, http_prefix,
domain_full, person_cache,
signing_priv_key_pem,
max_recent_posts,
translate,
session,
cached_webfingers,
port,
allow_deletion,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
theme_name, max_like_count,
cw_lists, dogwhistles,
min_images_for_accounts)
# update the index
id_str = edited_postid.split('/')[-1]
index_filename = \
acct_dir(base_dir, nickname, domain) + '/' + box_name + '.index'
if not text_in_file(id_str, index_filename):
try:
with open(index_filename, 'r+',
encoding='utf-8') as fp_index:
content = fp_index.read()
if id_str + '\n' not in content:
fp_index.seek(0, 0)
fp_index.write(id_str + '\n' + content)
except OSError as ex:
print('WARN: Failed to write index after edit ' +
index_filename + ' ' + str(ex))
def _convert_domains(self, calling_domain, referer_domain,
msg_str: str) -> str:
"""Convert domains to onion or i2p, depending upon who is asking
@ -19448,24 +19536,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate)
if message_json:
if edited_postid:
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
recent_posts_cache = self.server.recent_posts_cache
allow_local_network_access = \
self.server.allow_local_network_access
signing_priv_key_pem = \
self.server.signing_priv_key_pem
twitter_replacement_domain = \
self.server.twitter_replacement_domain
show_published_date_only = \
self.server.show_published_date_only
min_images_for_accounts = \
self.server.min_images_for_accounts
peertube_instances = \
self.server.peertube_instances
self._update_edited_post(self.server.base_dir,
nickname, self.server.domain,
message_json,
edited_published,
edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited public post ' +
str(message_json))
if fields['schedulePost']:
@ -19725,24 +19839,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate)
if message_json:
if edited_postid:
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
recent_posts_cache = self.server.recent_posts_cache
allow_local_network_access = \
self.server.allow_local_network_access
signing_priv_key_pem = \
self.server.signing_priv_key_pem
twitter_replacement_domain = \
self.server.twitter_replacement_domain
show_published_date_only = \
self.server.show_published_date_only
min_images_for_accounts = \
self.server.min_images_for_accounts
peertube_instances = \
self.server.peertube_instances
self._update_edited_post(self.server.base_dir,
nickname, self.server.domain,
message_json,
edited_published,
edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited unlisted post ' +
str(message_json))
@ -19808,24 +19948,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate)
if message_json:
if edited_postid:
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
recent_posts_cache = self.server.recent_posts_cache
allow_local_network_access = \
self.server.allow_local_network_access
signing_priv_key_pem = \
self.server.signing_priv_key_pem
twitter_replacement_domain = \
self.server.twitter_replacement_domain
show_published_date_only = \
self.server.show_published_date_only
min_images_for_accounts = \
self.server.min_images_for_accounts
peertube_instances = \
self.server.peertube_instances
self._update_edited_post(self.server.base_dir,
nickname, self.server.domain,
message_json,
edited_published,
edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited followers post ' +
str(message_json))
@ -19905,24 +20071,50 @@ class PubServer(BaseHTTPRequestHandler):
print('DEBUG: posting DM edited_postid ' +
str(edited_postid))
if edited_postid:
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
recent_posts_cache = self.server.recent_posts_cache
allow_local_network_access = \
self.server.allow_local_network_access
signing_priv_key_pem = \
self.server.signing_priv_key_pem
twitter_replacement_domain = \
self.server.twitter_replacement_domain
show_published_date_only = \
self.server.show_published_date_only
min_images_for_accounts = \
self.server.min_images_for_accounts
peertube_instances = \
self.server.peertube_instances
self._update_edited_post(self.server.base_dir,
nickname, self.server.domain,
message_json,
edited_published,
edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited dm post ' +
str(message_json))
@ -22012,6 +22204,9 @@ def run_daemon(map_format: str,
create_initial_last_seen(base_dir, http_prefix)
httpd.max_mentions = max_mentions
httpd.max_emoji = max_emoji
print('THREAD: Creating inbox queue')
httpd.thrInboxQueue = \
thread_with_trace(target=run_inbox_queue,

View File

@ -1137,25 +1137,25 @@ def _receive_update_to_question(recent_posts_cache: {}, message_json: {},
remove_post_from_cache(message_json, recent_posts_cache)
def _receive_edit_to_post(recent_posts_cache: {}, message_json: {},
base_dir: str,
nickname: str, domain: str,
max_mentions: int, max_emoji: int,
allow_local_network_access: bool,
debug: bool,
system_language: str, http_prefix: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
max_recent_posts: int, translate: {},
session, cached_webfingers: {}, port: int,
allow_deletion: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
peertube_instances: [],
theme_name: str, max_like_count: int,
cw_lists: {}, dogwhistles: {},
min_images_for_accounts: []) -> bool:
def receive_edit_to_post(recent_posts_cache: {}, message_json: {},
base_dir: str,
nickname: str, domain: str,
max_mentions: int, max_emoji: int,
allow_local_network_access: bool,
debug: bool,
system_language: str, http_prefix: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
max_recent_posts: int, translate: {},
session, cached_webfingers: {}, port: int,
allow_deletion: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
peertube_instances: [],
theme_name: str, max_like_count: int,
cw_lists: {}, dogwhistles: {},
min_images_for_accounts: []) -> bool:
"""A post was edited
"""
if not has_object_dict(message_json):
@ -1205,7 +1205,7 @@ def _receive_edit_to_post(recent_posts_cache: {}, message_json: {},
# has the content changed?
if post_json_object['object']['content'] == \
message_json['object']['content']:
# same content
# same content. Has the summary changed?
if 'summary' in post_json_object['object'] and \
'summary' in message_json['object']:
if post_json_object['object']['summary'] == \
@ -1329,23 +1329,23 @@ def _receive_update_activity(recent_posts_cache: {}, session, base_dir: str,
elif message_json['object']['type'] == 'Note':
if message_json['object'].get('id'):
domain_full = get_full_domain(domain, port)
if _receive_edit_to_post(recent_posts_cache, message_json,
base_dir, nickname, domain,
max_mentions, max_emoji,
allow_local_network_access,
debug, system_language, http_prefix,
domain_full, person_cache,
signing_priv_key_pem,
max_recent_posts, translate,
session, cached_webfingers, port,
allow_deletion,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
theme_name, max_like_count,
cw_lists, dogwhistles,
min_images_for_accounts):
if receive_edit_to_post(recent_posts_cache, message_json,
base_dir, nickname, domain,
max_mentions, max_emoji,
allow_local_network_access,
debug, system_language, http_prefix,
domain_full, person_cache,
signing_priv_key_pem,
max_recent_posts, translate,
session, cached_webfingers, port,
allow_deletion,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
theme_name, max_like_count,
cw_lists, dogwhistles,
min_images_for_accounts):
print('EDITPOST: received ' + message_json['object']['id'])
return True
else:

265
outbox.py
View File

@ -414,148 +414,157 @@ def post_message_to_outbox(session, translate: {},
post_id = None
if debug:
print('DEBUG: save_post_to_box')
if message_json['type'] != 'Upgrade':
outbox_name = 'outbox'
store_hash_tags(base_dir, post_to_nickname, domain,
http_prefix, domain_full,
message_json, translate)
is_edited_post = False
if message_json['type'] == 'Update' and \
message_json['object']['type'] == 'Note':
is_edited_post = True
message_json['type'] = 'Create'
# if this is a blog post or an event then save to its own box
if message_json['type'] == 'Create':
if has_object_dict(message_json):
if message_json['object'].get('type'):
if message_json['object']['type'] == 'Article':
outbox_name = 'tlblogs'
outbox_name = 'outbox'
saved_filename = \
save_post_to_box(base_dir,
http_prefix,
post_id,
post_to_nickname, domain_full,
message_json, outbox_name)
if not saved_filename:
print('WARN: post not saved to outbox ' + outbox_name)
return False
store_hash_tags(base_dir, post_to_nickname, domain,
http_prefix, domain_full,
message_json, translate)
update_speaker(base_dir, http_prefix,
post_to_nickname, domain, domain_full,
message_json, person_cache,
translate, message_json['actor'],
theme, system_language,
outbox_name)
# if this is a blog post or an event then save to its own box
if message_json['type'] == 'Create':
if has_object_dict(message_json):
if message_json['object'].get('type'):
if message_json['object']['type'] == 'Article':
outbox_name = 'tlblogs'
# save all instance blogs to the news actor
if post_to_nickname != 'news' and outbox_name == 'tlblogs':
if '/' in saved_filename:
if is_featured_writer(base_dir, post_to_nickname, domain):
saved_post_id = saved_filename.split('/')[-1]
blogs_dir = \
base_dir + '/accounts/news@' + domain + '/tlblogs'
if not os.path.isdir(blogs_dir):
os.mkdir(blogs_dir)
copyfile(saved_filename, blogs_dir + '/' + saved_post_id)
inbox_update_index('tlblogs', base_dir,
'news@' + domain,
saved_filename, debug)
saved_filename = \
save_post_to_box(base_dir,
http_prefix,
post_id,
post_to_nickname, domain_full,
message_json, outbox_name)
if not saved_filename:
print('WARN: post not saved to outbox ' + outbox_name)
return False
# clear the citations file if it exists
citations_filename = \
base_dir + '/accounts/' + \
post_to_nickname + '@' + domain + '/.citations.txt'
if os.path.isfile(citations_filename):
try:
os.remove(citations_filename)
except OSError:
print('EX: post_message_to_outbox unable to delete ' +
citations_filename)
update_speaker(base_dir, http_prefix,
post_to_nickname, domain, domain_full,
message_json, person_cache,
translate, message_json['actor'],
theme, system_language,
outbox_name)
# The following activity types get added to the index files
indexed_activities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce', 'ChatMessage'
)
if message_json['type'] in indexed_activities:
indexes = [outbox_name, "inbox"]
self_actor = \
local_actor_url(http_prefix, post_to_nickname, domain_full)
for box_name_index in indexes:
if not box_name_index:
continue
# save all instance blogs to the news actor
if post_to_nickname != 'news' and outbox_name == 'tlblogs':
if '/' in saved_filename:
if is_featured_writer(base_dir, post_to_nickname, domain):
saved_post_id = saved_filename.split('/')[-1]
blogs_dir = \
base_dir + '/accounts/news@' + domain + '/tlblogs'
if not os.path.isdir(blogs_dir):
os.mkdir(blogs_dir)
copyfile(saved_filename, blogs_dir + '/' + saved_post_id)
inbox_update_index('tlblogs', base_dir,
'news@' + domain,
saved_filename, debug)
# should this also go to the media timeline?
if box_name_index == 'inbox':
if is_image_media(session, base_dir, http_prefix,
post_to_nickname, domain,
message_json,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
bold_reading):
inbox_update_index('tlmedia', base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
# clear the citations file if it exists
citations_filename = \
base_dir + '/accounts/' + \
post_to_nickname + '@' + domain + '/.citations.txt'
if os.path.isfile(citations_filename):
try:
os.remove(citations_filename)
except OSError:
print('EX: post_message_to_outbox unable to delete ' +
citations_filename)
if box_name_index == 'inbox' and outbox_name == 'tlblogs':
continue
# The following activity types get added to the index files
indexed_activities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce', 'ChatMessage'
)
if message_json['type'] in indexed_activities:
indexes = [outbox_name, "inbox"]
self_actor = \
local_actor_url(http_prefix, post_to_nickname, domain_full)
for box_name_index in indexes:
if not box_name_index:
continue
# avoid duplicates of the message if already going
# back to the inbox of the same account
if self_actor not in message_json['to']:
# show sent post within the inbox,
# as is the typical convention
inbox_update_index(box_name_index, base_dir,
# should this also go to the media timeline?
if box_name_index == 'inbox':
if is_image_media(session, base_dir, http_prefix,
post_to_nickname, domain,
message_json,
yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
bold_reading):
inbox_update_index('tlmedia', base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
# regenerate the html
use_cache_only = False
page_number = 1
show_individual_post_icons = True
manually_approve_followers = \
follower_approval_active(base_dir,
post_to_nickname, domain)
timezone = \
get_account_timezone(base_dir,
if box_name_index == 'inbox' and outbox_name == 'tlblogs':
continue
# avoid duplicates of the message if already going
# back to the inbox of the same account
if self_actor not in message_json['to']:
# show sent post within the inbox,
# as is the typical convention
inbox_update_index(box_name_index, base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
# regenerate the html
use_cache_only = False
page_number = 1
show_individual_post_icons = True
manually_approve_followers = \
follower_approval_active(base_dir,
post_to_nickname, domain)
mitm = False
if os.path.isfile(saved_filename.replace('.json', '') +
'.mitm'):
mitm = True
minimize_all_images = False
if post_to_nickname in min_images_for_accounts:
minimize_all_images = True
individual_post_as_html(signing_priv_key_pem,
False, recent_posts_cache,
max_recent_posts,
translate, page_number,
base_dir, session,
cached_webfingers,
person_cache,
post_to_nickname, domain, port,
message_json, None, True,
allow_deletion,
http_prefix, __version__,
box_name_index,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
allow_local_network_access,
theme, system_language,
max_like_count,
box_name_index != 'dm',
show_individual_post_icons,
manually_approve_followers,
False, True, use_cache_only,
cw_lists, lists_enabled,
timezone, mitm,
bold_reading, dogwhistles,
minimize_all_images)
timezone = \
get_account_timezone(base_dir,
post_to_nickname, domain)
mitm = False
if os.path.isfile(saved_filename.replace('.json', '') +
'.mitm'):
mitm = True
minimize_all_images = False
if post_to_nickname in min_images_for_accounts:
minimize_all_images = True
individual_post_as_html(signing_priv_key_pem,
False, recent_posts_cache,
max_recent_posts,
translate, page_number,
base_dir, session,
cached_webfingers,
person_cache,
post_to_nickname, domain, port,
message_json, None, True,
allow_deletion,
http_prefix, __version__,
box_name_index,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
allow_local_network_access,
theme, system_language,
max_like_count,
box_name_index != 'dm',
show_individual_post_icons,
manually_approve_followers,
False, True, use_cache_only,
cw_lists, lists_enabled,
timezone, mitm,
bold_reading, dogwhistles,
minimize_all_images)
if is_edited_post:
message_json['type'] = 'Update'
if outbox_announce(recent_posts_cache,
base_dir, message_json, debug):