diff --git a/daemon.py b/daemon.py index b45e22221..53a545e3f 100644 --- a/daemon.py +++ b/daemon.py @@ -114,6 +114,7 @@ from inbox import run_inbox_queue from inbox import run_inbox_queue_watchdog from inbox import save_post_to_inbox_queue from inbox import populate_replies +from inbox import receive_edit_to_post from follow import follower_approval_active from follow import is_following_actor from follow import get_following_feed @@ -439,6 +440,93 @@ SHARES_PER_PAGE = 12 class PubServer(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' + def _update_edited_post(self, base_dir: str, + nickname: str, domain: str, + message_json: {}, + edited_published: str, + edited_postid: str, + recent_posts_cache: {}, + box_name: str, + max_mentions: int, max_emoji: int, + allow_local_network_access: bool, + debug: bool, + system_language: str, http_prefix: str, + domain_full: str, person_cache: {}, + signing_priv_key_pem: str, + max_recent_posts: int, translate: {}, + session, cached_webfingers: {}, port: int, + allow_deletion: bool, + yt_replace_domain: str, + twitter_replacement_domain: str, + show_published_date_only: bool, + peertube_instances: [], + theme_name: str, max_like_count: int, + cw_lists: {}, dogwhistles: {}, + min_images_for_accounts: []) -> None: + """When an edited post is created this assigns + a published and updated date to it, and uses + the previous id + """ + edited_updated = \ + message_json['object']['published'] + if edited_published: + message_json['published'] = \ + edited_published + message_json['object']['published'] = \ + edited_published + message_json['id'] = \ + edited_postid + '/activity' + message_json['object']['id'] = \ + edited_postid + message_json['object']['url'] = \ + edited_postid + message_json['updated'] = \ + edited_updated + message_json['object']['updated'] = \ + edited_updated + message_json['type'] = 'Update' + + message_json2 = message_json.copy() + receive_edit_to_post(recent_posts_cache, + message_json2, + base_dir, + nickname, domain, + max_mentions, max_emoji, + allow_local_network_access, + debug, + system_language, http_prefix, + domain_full, person_cache, + signing_priv_key_pem, + max_recent_posts, + translate, + session, + cached_webfingers, + port, + allow_deletion, + yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + theme_name, max_like_count, + cw_lists, dogwhistles, + min_images_for_accounts) + + # update the index + id_str = edited_postid.split('/')[-1] + index_filename = \ + acct_dir(base_dir, nickname, domain) + '/' + box_name + '.index' + if not text_in_file(id_str, index_filename): + try: + with open(index_filename, 'r+', + encoding='utf-8') as fp_index: + content = fp_index.read() + if id_str + '\n' not in content: + fp_index.seek(0, 0) + fp_index.write(id_str + '\n' + content) + except OSError as ex: + print('WARN: Failed to write index after edit ' + + index_filename + ' ' + str(ex)) + def _convert_domains(self, calling_domain, referer_domain, msg_str: str) -> str: """Convert domains to onion or i2p, depending upon who is asking @@ -19448,24 +19536,50 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate) if message_json: if edited_postid: - edited_updated = \ - message_json['object']['published'] - if edited_published: - message_json['published'] = \ - edited_published - message_json['object']['published'] = \ - edited_published - message_json['id'] = \ - edited_postid + '/activity' - message_json['object']['id'] = \ - edited_postid - message_json['object']['url'] = \ - edited_postid - message_json['updated'] = \ - edited_updated - message_json['object']['updated'] = \ - edited_updated - message_json['type'] = 'Update' + recent_posts_cache = self.server.recent_posts_cache + allow_local_network_access = \ + self.server.allow_local_network_access + signing_priv_key_pem = \ + self.server.signing_priv_key_pem + twitter_replacement_domain = \ + self.server.twitter_replacement_domain + show_published_date_only = \ + self.server.show_published_date_only + min_images_for_accounts = \ + self.server.min_images_for_accounts + peertube_instances = \ + self.server.peertube_instances + self._update_edited_post(self.server.base_dir, + nickname, self.server.domain, + message_json, + edited_published, + edited_postid, + recent_posts_cache, + 'outbox', + self.server.max_mentions, + self.server.max_emoji, + allow_local_network_access, + self.server.debug, + self.server.system_language, + self.server.http_prefix, + self.server.domain_full, + self.server.person_cache, + signing_priv_key_pem, + self.server.max_recent_posts, + self.server.translate, + curr_session, + self.server.cached_webfingers, + self.server.port, + self.server.allow_deletion, + self.server.yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + self.server.theme_name, + self.server.max_like_count, + self.server.cw_lists, + self.server.dogwhistles, + min_images_for_accounts) print('DEBUG: sending edited public post ' + str(message_json)) if fields['schedulePost']: @@ -19725,24 +19839,50 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate) if message_json: if edited_postid: - edited_updated = \ - message_json['object']['published'] - if edited_published: - message_json['published'] = \ - edited_published - message_json['object']['published'] = \ - edited_published - message_json['id'] = \ - edited_postid + '/activity' - message_json['object']['id'] = \ - edited_postid - message_json['object']['url'] = \ - edited_postid - message_json['updated'] = \ - edited_updated - message_json['object']['updated'] = \ - edited_updated - message_json['type'] = 'Update' + recent_posts_cache = self.server.recent_posts_cache + allow_local_network_access = \ + self.server.allow_local_network_access + signing_priv_key_pem = \ + self.server.signing_priv_key_pem + twitter_replacement_domain = \ + self.server.twitter_replacement_domain + show_published_date_only = \ + self.server.show_published_date_only + min_images_for_accounts = \ + self.server.min_images_for_accounts + peertube_instances = \ + self.server.peertube_instances + self._update_edited_post(self.server.base_dir, + nickname, self.server.domain, + message_json, + edited_published, + edited_postid, + recent_posts_cache, + 'outbox', + self.server.max_mentions, + self.server.max_emoji, + allow_local_network_access, + self.server.debug, + self.server.system_language, + self.server.http_prefix, + self.server.domain_full, + self.server.person_cache, + signing_priv_key_pem, + self.server.max_recent_posts, + self.server.translate, + curr_session, + self.server.cached_webfingers, + self.server.port, + self.server.allow_deletion, + self.server.yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + self.server.theme_name, + self.server.max_like_count, + self.server.cw_lists, + self.server.dogwhistles, + min_images_for_accounts) print('DEBUG: sending edited unlisted post ' + str(message_json)) @@ -19808,24 +19948,50 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate) if message_json: if edited_postid: - edited_updated = \ - message_json['object']['published'] - if edited_published: - message_json['published'] = \ - edited_published - message_json['object']['published'] = \ - edited_published - message_json['id'] = \ - edited_postid + '/activity' - message_json['object']['id'] = \ - edited_postid - message_json['object']['url'] = \ - edited_postid - message_json['updated'] = \ - edited_updated - message_json['object']['updated'] = \ - edited_updated - message_json['type'] = 'Update' + recent_posts_cache = self.server.recent_posts_cache + allow_local_network_access = \ + self.server.allow_local_network_access + signing_priv_key_pem = \ + self.server.signing_priv_key_pem + twitter_replacement_domain = \ + self.server.twitter_replacement_domain + show_published_date_only = \ + self.server.show_published_date_only + min_images_for_accounts = \ + self.server.min_images_for_accounts + peertube_instances = \ + self.server.peertube_instances + self._update_edited_post(self.server.base_dir, + nickname, self.server.domain, + message_json, + edited_published, + edited_postid, + recent_posts_cache, + 'outbox', + self.server.max_mentions, + self.server.max_emoji, + allow_local_network_access, + self.server.debug, + self.server.system_language, + self.server.http_prefix, + self.server.domain_full, + self.server.person_cache, + signing_priv_key_pem, + self.server.max_recent_posts, + self.server.translate, + curr_session, + self.server.cached_webfingers, + self.server.port, + self.server.allow_deletion, + self.server.yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + self.server.theme_name, + self.server.max_like_count, + self.server.cw_lists, + self.server.dogwhistles, + min_images_for_accounts) print('DEBUG: sending edited followers post ' + str(message_json)) @@ -19905,24 +20071,50 @@ class PubServer(BaseHTTPRequestHandler): print('DEBUG: posting DM edited_postid ' + str(edited_postid)) if edited_postid: - edited_updated = \ - message_json['object']['published'] - if edited_published: - message_json['published'] = \ - edited_published - message_json['object']['published'] = \ - edited_published - message_json['id'] = \ - edited_postid + '/activity' - message_json['object']['id'] = \ - edited_postid - message_json['object']['url'] = \ - edited_postid - message_json['updated'] = \ - edited_updated - message_json['object']['updated'] = \ - edited_updated - message_json['type'] = 'Update' + recent_posts_cache = self.server.recent_posts_cache + allow_local_network_access = \ + self.server.allow_local_network_access + signing_priv_key_pem = \ + self.server.signing_priv_key_pem + twitter_replacement_domain = \ + self.server.twitter_replacement_domain + show_published_date_only = \ + self.server.show_published_date_only + min_images_for_accounts = \ + self.server.min_images_for_accounts + peertube_instances = \ + self.server.peertube_instances + self._update_edited_post(self.server.base_dir, + nickname, self.server.domain, + message_json, + edited_published, + edited_postid, + recent_posts_cache, + 'outbox', + self.server.max_mentions, + self.server.max_emoji, + allow_local_network_access, + self.server.debug, + self.server.system_language, + self.server.http_prefix, + self.server.domain_full, + self.server.person_cache, + signing_priv_key_pem, + self.server.max_recent_posts, + self.server.translate, + curr_session, + self.server.cached_webfingers, + self.server.port, + self.server.allow_deletion, + self.server.yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + self.server.theme_name, + self.server.max_like_count, + self.server.cw_lists, + self.server.dogwhistles, + min_images_for_accounts) print('DEBUG: sending edited dm post ' + str(message_json)) @@ -22012,6 +22204,9 @@ def run_daemon(map_format: str, create_initial_last_seen(base_dir, http_prefix) + httpd.max_mentions = max_mentions + httpd.max_emoji = max_emoji + print('THREAD: Creating inbox queue') httpd.thrInboxQueue = \ thread_with_trace(target=run_inbox_queue, diff --git a/inbox.py b/inbox.py index 0862f269e..813b1b709 100644 --- a/inbox.py +++ b/inbox.py @@ -1137,25 +1137,25 @@ def _receive_update_to_question(recent_posts_cache: {}, message_json: {}, remove_post_from_cache(message_json, recent_posts_cache) -def _receive_edit_to_post(recent_posts_cache: {}, message_json: {}, - base_dir: str, - nickname: str, domain: str, - max_mentions: int, max_emoji: int, - allow_local_network_access: bool, - debug: bool, - system_language: str, http_prefix: str, - domain_full: str, person_cache: {}, - signing_priv_key_pem: str, - max_recent_posts: int, translate: {}, - session, cached_webfingers: {}, port: int, - allow_deletion: bool, - yt_replace_domain: str, - twitter_replacement_domain: str, - show_published_date_only: bool, - peertube_instances: [], - theme_name: str, max_like_count: int, - cw_lists: {}, dogwhistles: {}, - min_images_for_accounts: []) -> bool: +def receive_edit_to_post(recent_posts_cache: {}, message_json: {}, + base_dir: str, + nickname: str, domain: str, + max_mentions: int, max_emoji: int, + allow_local_network_access: bool, + debug: bool, + system_language: str, http_prefix: str, + domain_full: str, person_cache: {}, + signing_priv_key_pem: str, + max_recent_posts: int, translate: {}, + session, cached_webfingers: {}, port: int, + allow_deletion: bool, + yt_replace_domain: str, + twitter_replacement_domain: str, + show_published_date_only: bool, + peertube_instances: [], + theme_name: str, max_like_count: int, + cw_lists: {}, dogwhistles: {}, + min_images_for_accounts: []) -> bool: """A post was edited """ if not has_object_dict(message_json): @@ -1205,7 +1205,7 @@ def _receive_edit_to_post(recent_posts_cache: {}, message_json: {}, # has the content changed? if post_json_object['object']['content'] == \ message_json['object']['content']: - # same content + # same content. Has the summary changed? if 'summary' in post_json_object['object'] and \ 'summary' in message_json['object']: if post_json_object['object']['summary'] == \ @@ -1329,23 +1329,23 @@ def _receive_update_activity(recent_posts_cache: {}, session, base_dir: str, elif message_json['object']['type'] == 'Note': if message_json['object'].get('id'): domain_full = get_full_domain(domain, port) - if _receive_edit_to_post(recent_posts_cache, message_json, - base_dir, nickname, domain, - max_mentions, max_emoji, - allow_local_network_access, - debug, system_language, http_prefix, - domain_full, person_cache, - signing_priv_key_pem, - max_recent_posts, translate, - session, cached_webfingers, port, - allow_deletion, - yt_replace_domain, - twitter_replacement_domain, - show_published_date_only, - peertube_instances, - theme_name, max_like_count, - cw_lists, dogwhistles, - min_images_for_accounts): + if receive_edit_to_post(recent_posts_cache, message_json, + base_dir, nickname, domain, + max_mentions, max_emoji, + allow_local_network_access, + debug, system_language, http_prefix, + domain_full, person_cache, + signing_priv_key_pem, + max_recent_posts, translate, + session, cached_webfingers, port, + allow_deletion, + yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + theme_name, max_like_count, + cw_lists, dogwhistles, + min_images_for_accounts): print('EDITPOST: received ' + message_json['object']['id']) return True else: diff --git a/outbox.py b/outbox.py index 9e86b8112..4132a808f 100644 --- a/outbox.py +++ b/outbox.py @@ -414,148 +414,157 @@ def post_message_to_outbox(session, translate: {}, post_id = None if debug: print('DEBUG: save_post_to_box') - if message_json['type'] != 'Upgrade': - outbox_name = 'outbox' - store_hash_tags(base_dir, post_to_nickname, domain, - http_prefix, domain_full, - message_json, translate) + is_edited_post = False + if message_json['type'] == 'Update' and \ + message_json['object']['type'] == 'Note': + is_edited_post = True + message_json['type'] = 'Create' - # if this is a blog post or an event then save to its own box - if message_json['type'] == 'Create': - if has_object_dict(message_json): - if message_json['object'].get('type'): - if message_json['object']['type'] == 'Article': - outbox_name = 'tlblogs' + outbox_name = 'outbox' - saved_filename = \ - save_post_to_box(base_dir, - http_prefix, - post_id, - post_to_nickname, domain_full, - message_json, outbox_name) - if not saved_filename: - print('WARN: post not saved to outbox ' + outbox_name) - return False + store_hash_tags(base_dir, post_to_nickname, domain, + http_prefix, domain_full, + message_json, translate) - update_speaker(base_dir, http_prefix, - post_to_nickname, domain, domain_full, - message_json, person_cache, - translate, message_json['actor'], - theme, system_language, - outbox_name) + # if this is a blog post or an event then save to its own box + if message_json['type'] == 'Create': + if has_object_dict(message_json): + if message_json['object'].get('type'): + if message_json['object']['type'] == 'Article': + outbox_name = 'tlblogs' - # save all instance blogs to the news actor - if post_to_nickname != 'news' and outbox_name == 'tlblogs': - if '/' in saved_filename: - if is_featured_writer(base_dir, post_to_nickname, domain): - saved_post_id = saved_filename.split('/')[-1] - blogs_dir = \ - base_dir + '/accounts/news@' + domain + '/tlblogs' - if not os.path.isdir(blogs_dir): - os.mkdir(blogs_dir) - copyfile(saved_filename, blogs_dir + '/' + saved_post_id) - inbox_update_index('tlblogs', base_dir, - 'news@' + domain, - saved_filename, debug) + saved_filename = \ + save_post_to_box(base_dir, + http_prefix, + post_id, + post_to_nickname, domain_full, + message_json, outbox_name) + if not saved_filename: + print('WARN: post not saved to outbox ' + outbox_name) + return False - # clear the citations file if it exists - citations_filename = \ - base_dir + '/accounts/' + \ - post_to_nickname + '@' + domain + '/.citations.txt' - if os.path.isfile(citations_filename): - try: - os.remove(citations_filename) - except OSError: - print('EX: post_message_to_outbox unable to delete ' + - citations_filename) + update_speaker(base_dir, http_prefix, + post_to_nickname, domain, domain_full, + message_json, person_cache, + translate, message_json['actor'], + theme, system_language, + outbox_name) - # The following activity types get added to the index files - indexed_activities = ( - 'Create', 'Question', 'Note', 'EncryptedMessage', 'Article', - 'Patch', 'Announce', 'ChatMessage' - ) - if message_json['type'] in indexed_activities: - indexes = [outbox_name, "inbox"] - self_actor = \ - local_actor_url(http_prefix, post_to_nickname, domain_full) - for box_name_index in indexes: - if not box_name_index: - continue + # save all instance blogs to the news actor + if post_to_nickname != 'news' and outbox_name == 'tlblogs': + if '/' in saved_filename: + if is_featured_writer(base_dir, post_to_nickname, domain): + saved_post_id = saved_filename.split('/')[-1] + blogs_dir = \ + base_dir + '/accounts/news@' + domain + '/tlblogs' + if not os.path.isdir(blogs_dir): + os.mkdir(blogs_dir) + copyfile(saved_filename, blogs_dir + '/' + saved_post_id) + inbox_update_index('tlblogs', base_dir, + 'news@' + domain, + saved_filename, debug) - # should this also go to the media timeline? - if box_name_index == 'inbox': - if is_image_media(session, base_dir, http_prefix, - post_to_nickname, domain, - message_json, - yt_replace_domain, - twitter_replacement_domain, - allow_local_network_access, - recent_posts_cache, debug, - system_language, - domain_full, person_cache, - signing_priv_key_pem, - bold_reading): - inbox_update_index('tlmedia', base_dir, - post_to_nickname + '@' + domain, - saved_filename, debug) + # clear the citations file if it exists + citations_filename = \ + base_dir + '/accounts/' + \ + post_to_nickname + '@' + domain + '/.citations.txt' + if os.path.isfile(citations_filename): + try: + os.remove(citations_filename) + except OSError: + print('EX: post_message_to_outbox unable to delete ' + + citations_filename) - if box_name_index == 'inbox' and outbox_name == 'tlblogs': - continue + # The following activity types get added to the index files + indexed_activities = ( + 'Create', 'Question', 'Note', 'EncryptedMessage', 'Article', + 'Patch', 'Announce', 'ChatMessage' + ) + if message_json['type'] in indexed_activities: + indexes = [outbox_name, "inbox"] + self_actor = \ + local_actor_url(http_prefix, post_to_nickname, domain_full) + for box_name_index in indexes: + if not box_name_index: + continue - # avoid duplicates of the message if already going - # back to the inbox of the same account - if self_actor not in message_json['to']: - # show sent post within the inbox, - # as is the typical convention - inbox_update_index(box_name_index, base_dir, + # should this also go to the media timeline? + if box_name_index == 'inbox': + if is_image_media(session, base_dir, http_prefix, + post_to_nickname, domain, + message_json, + yt_replace_domain, + twitter_replacement_domain, + allow_local_network_access, + recent_posts_cache, debug, + system_language, + domain_full, person_cache, + signing_priv_key_pem, + bold_reading): + inbox_update_index('tlmedia', base_dir, post_to_nickname + '@' + domain, saved_filename, debug) - # regenerate the html - use_cache_only = False - page_number = 1 - show_individual_post_icons = True - manually_approve_followers = \ - follower_approval_active(base_dir, - post_to_nickname, domain) - timezone = \ - get_account_timezone(base_dir, + if box_name_index == 'inbox' and outbox_name == 'tlblogs': + continue + + # avoid duplicates of the message if already going + # back to the inbox of the same account + if self_actor not in message_json['to']: + # show sent post within the inbox, + # as is the typical convention + inbox_update_index(box_name_index, base_dir, + post_to_nickname + '@' + domain, + saved_filename, debug) + + # regenerate the html + use_cache_only = False + page_number = 1 + show_individual_post_icons = True + manually_approve_followers = \ + follower_approval_active(base_dir, post_to_nickname, domain) - mitm = False - if os.path.isfile(saved_filename.replace('.json', '') + - '.mitm'): - mitm = True - minimize_all_images = False - if post_to_nickname in min_images_for_accounts: - minimize_all_images = True - individual_post_as_html(signing_priv_key_pem, - False, recent_posts_cache, - max_recent_posts, - translate, page_number, - base_dir, session, - cached_webfingers, - person_cache, - post_to_nickname, domain, port, - message_json, None, True, - allow_deletion, - http_prefix, __version__, - box_name_index, - yt_replace_domain, - twitter_replacement_domain, - show_published_date_only, - peertube_instances, - allow_local_network_access, - theme, system_language, - max_like_count, - box_name_index != 'dm', - show_individual_post_icons, - manually_approve_followers, - False, True, use_cache_only, - cw_lists, lists_enabled, - timezone, mitm, - bold_reading, dogwhistles, - minimize_all_images) + timezone = \ + get_account_timezone(base_dir, + post_to_nickname, domain) + mitm = False + if os.path.isfile(saved_filename.replace('.json', '') + + '.mitm'): + mitm = True + minimize_all_images = False + if post_to_nickname in min_images_for_accounts: + minimize_all_images = True + individual_post_as_html(signing_priv_key_pem, + False, recent_posts_cache, + max_recent_posts, + translate, page_number, + base_dir, session, + cached_webfingers, + person_cache, + post_to_nickname, domain, port, + message_json, None, True, + allow_deletion, + http_prefix, __version__, + box_name_index, + yt_replace_domain, + twitter_replacement_domain, + show_published_date_only, + peertube_instances, + allow_local_network_access, + theme, system_language, + max_like_count, + box_name_index != 'dm', + show_individual_post_icons, + manually_approve_followers, + False, True, use_cache_only, + cw_lists, lists_enabled, + timezone, mitm, + bold_reading, dogwhistles, + minimize_all_images) + + if is_edited_post: + message_json['type'] = 'Update' if outbox_announce(recent_posts_cache, base_dir, message_json, debug):