indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Merge branch 'main' of gitlab.com:bashrc2/epicyon

merge-requests/30/head
Bob Mottram 2022-11-17 20:27:35 +00:00
parent 869d2fbda4 c65be9094b
commit fb38442ed3
3 changed files with 441 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

339
daemon.py
View File

@ -114,6 +114,7 @@ from inbox import run_inbox_queue
from inbox import run_inbox_queue_watchdog from inbox import run_inbox_queue_watchdog
from inbox import save_post_to_inbox_queue from inbox import save_post_to_inbox_queue
from inbox import populate_replies from inbox import populate_replies
from inbox import receive_edit_to_post
from follow import follower_approval_active from follow import follower_approval_active
from follow import is_following_actor from follow import is_following_actor
from follow import get_following_feed from follow import get_following_feed
@ -439,6 +440,93 @@ SHARES_PER_PAGE = 12
class PubServer(BaseHTTPRequestHandler): class PubServer(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1' protocol_version = 'HTTP/1.1'
def _update_edited_post(self, base_dir: str,
nickname: str, domain: str,
message_json: {},
edited_published: str,
edited_postid: str,
recent_posts_cache: {},
box_name: str,
max_mentions: int, max_emoji: int,
allow_local_network_access: bool,
debug: bool,
system_language: str, http_prefix: str,
domain_full: str, person_cache: {},
signing_priv_key_pem: str,
max_recent_posts: int, translate: {},
session, cached_webfingers: {}, port: int,
allow_deletion: bool,
yt_replace_domain: str,
twitter_replacement_domain: str,
show_published_date_only: bool,
peertube_instances: [],
theme_name: str, max_like_count: int,
cw_lists: {}, dogwhistles: {},
min_images_for_accounts: []) -> None:
"""When an edited post is created this assigns
a published and updated date to it, and uses
the previous id
"""
edited_updated = \
message_json['object']['published']
if edited_published:
message_json['published'] = \
edited_published
message_json['object']['published'] = \
edited_published
message_json['id'] = \
edited_postid + '/activity'
message_json['object']['id'] = \
edited_postid
message_json['object']['url'] = \
edited_postid
message_json['updated'] = \
edited_updated
message_json['object']['updated'] = \
edited_updated
message_json['type'] = 'Update'
message_json2 = message_json.copy()
receive_edit_to_post(recent_posts_cache,
message_json2,
base_dir,
nickname, domain,
max_mentions, max_emoji,
allow_local_network_access,
debug,
system_language, http_prefix,
domain_full, person_cache,
signing_priv_key_pem,
max_recent_posts,
translate,
session,
cached_webfingers,
port,
allow_deletion,
yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
theme_name, max_like_count,
cw_lists, dogwhistles,
min_images_for_accounts)
# update the index
id_str = edited_postid.split('/')[-1]
index_filename = \
acct_dir(base_dir, nickname, domain) + '/' + box_name + '.index'
if not text_in_file(id_str, index_filename):
try:
with open(index_filename, 'r+',
encoding='utf-8') as fp_index:
content = fp_index.read()
if id_str + '\n' not in content:
fp_index.seek(0, 0)
fp_index.write(id_str + '\n' + content)
except OSError as ex:
print('WARN: Failed to write index after edit ' +
index_filename + ' ' + str(ex))
def _convert_domains(self, calling_domain, referer_domain, def _convert_domains(self, calling_domain, referer_domain,
msg_str: str) -> str: msg_str: str) -> str:
"""Convert domains to onion or i2p, depending upon who is asking """Convert domains to onion or i2p, depending upon who is asking
@ -19448,24 +19536,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate) self.server.translate)
if message_json: if message_json:
if edited_postid: if edited_postid:
edited_updated = \ recent_posts_cache = self.server.recent_posts_cache
message_json['object']['published'] allow_local_network_access = \
if edited_published: self.server.allow_local_network_access
message_json['published'] = \ signing_priv_key_pem = \
edited_published self.server.signing_priv_key_pem
message_json['object']['published'] = \ twitter_replacement_domain = \
edited_published self.server.twitter_replacement_domain
message_json['id'] = \ show_published_date_only = \
edited_postid + '/activity' self.server.show_published_date_only
message_json['object']['id'] = \ min_images_for_accounts = \
edited_postid self.server.min_images_for_accounts
message_json['object']['url'] = \ peertube_instances = \
edited_postid self.server.peertube_instances
message_json['updated'] = \ self._update_edited_post(self.server.base_dir,
edited_updated nickname, self.server.domain,
message_json['object']['updated'] = \ message_json,
edited_updated edited_published,
message_json['type'] = 'Update' edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited public post ' + print('DEBUG: sending edited public post ' +
str(message_json)) str(message_json))
if fields['schedulePost']: if fields['schedulePost']:
@ -19725,24 +19839,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate) self.server.translate)
if message_json: if message_json:
if edited_postid: if edited_postid:
edited_updated = \ recent_posts_cache = self.server.recent_posts_cache
message_json['object']['published'] allow_local_network_access = \
if edited_published: self.server.allow_local_network_access
message_json['published'] = \ signing_priv_key_pem = \
edited_published self.server.signing_priv_key_pem
message_json['object']['published'] = \ twitter_replacement_domain = \
edited_published self.server.twitter_replacement_domain
message_json['id'] = \ show_published_date_only = \
edited_postid + '/activity' self.server.show_published_date_only
message_json['object']['id'] = \ min_images_for_accounts = \
edited_postid self.server.min_images_for_accounts
message_json['object']['url'] = \ peertube_instances = \
edited_postid self.server.peertube_instances
message_json['updated'] = \ self._update_edited_post(self.server.base_dir,
edited_updated nickname, self.server.domain,
message_json['object']['updated'] = \ message_json,
edited_updated edited_published,
message_json['type'] = 'Update' edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited unlisted post ' + print('DEBUG: sending edited unlisted post ' +
str(message_json)) str(message_json))
@ -19808,24 +19948,50 @@ class PubServer(BaseHTTPRequestHandler):
self.server.translate) self.server.translate)
if message_json: if message_json:
if edited_postid: if edited_postid:
edited_updated = \ recent_posts_cache = self.server.recent_posts_cache
message_json['object']['published'] allow_local_network_access = \
if edited_published: self.server.allow_local_network_access
message_json['published'] = \ signing_priv_key_pem = \
edited_published self.server.signing_priv_key_pem
message_json['object']['published'] = \ twitter_replacement_domain = \
edited_published self.server.twitter_replacement_domain
message_json['id'] = \ show_published_date_only = \
edited_postid + '/activity' self.server.show_published_date_only
message_json['object']['id'] = \ min_images_for_accounts = \
edited_postid self.server.min_images_for_accounts
message_json['object']['url'] = \ peertube_instances = \
edited_postid self.server.peertube_instances
message_json['updated'] = \ self._update_edited_post(self.server.base_dir,
edited_updated nickname, self.server.domain,
message_json['object']['updated'] = \ message_json,
edited_updated edited_published,
message_json['type'] = 'Update' edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited followers post ' + print('DEBUG: sending edited followers post ' +
str(message_json)) str(message_json))
@ -19905,24 +20071,50 @@ class PubServer(BaseHTTPRequestHandler):
print('DEBUG: posting DM edited_postid ' + print('DEBUG: posting DM edited_postid ' +
str(edited_postid)) str(edited_postid))
if edited_postid: if edited_postid:
edited_updated = \ recent_posts_cache = self.server.recent_posts_cache
message_json['object']['published'] allow_local_network_access = \
if edited_published: self.server.allow_local_network_access
message_json['published'] = \ signing_priv_key_pem = \
edited_published self.server.signing_priv_key_pem
message_json['object']['published'] = \ twitter_replacement_domain = \
edited_published self.server.twitter_replacement_domain
message_json['id'] = \ show_published_date_only = \
edited_postid + '/activity' self.server.show_published_date_only
message_json['object']['id'] = \ min_images_for_accounts = \
edited_postid self.server.min_images_for_accounts
message_json['object']['url'] = \ peertube_instances = \
edited_postid self.server.peertube_instances
message_json['updated'] = \ self._update_edited_post(self.server.base_dir,
edited_updated nickname, self.server.domain,
message_json['object']['updated'] = \ message_json,
edited_updated edited_published,
message_json['type'] = 'Update' edited_postid,
recent_posts_cache,
'outbox',
self.server.max_mentions,
self.server.max_emoji,
allow_local_network_access,
self.server.debug,
self.server.system_language,
self.server.http_prefix,
self.server.domain_full,
self.server.person_cache,
signing_priv_key_pem,
self.server.max_recent_posts,
self.server.translate,
curr_session,
self.server.cached_webfingers,
self.server.port,
self.server.allow_deletion,
self.server.yt_replace_domain,
twitter_replacement_domain,
show_published_date_only,
peertube_instances,
self.server.theme_name,
self.server.max_like_count,
self.server.cw_lists,
self.server.dogwhistles,
min_images_for_accounts)
print('DEBUG: sending edited dm post ' + print('DEBUG: sending edited dm post ' +
str(message_json)) str(message_json))
@ -22012,6 +22204,9 @@ def run_daemon(map_format: str,
create_initial_last_seen(base_dir, http_prefix) create_initial_last_seen(base_dir, http_prefix)
httpd.max_mentions = max_mentions
httpd.max_emoji = max_emoji
print('THREAD: Creating inbox queue') print('THREAD: Creating inbox queue')
httpd.thrInboxQueue = \ httpd.thrInboxQueue = \
thread_with_trace(target=run_inbox_queue, thread_with_trace(target=run_inbox_queue,

74
inbox.py
View File

@ -1137,25 +1137,25 @@ def _receive_update_to_question(recent_posts_cache: {}, message_json: {},
remove_post_from_cache(message_json, recent_posts_cache) remove_post_from_cache(message_json, recent_posts_cache)
def _receive_edit_to_post(recent_posts_cache: {}, message_json: {}, def receive_edit_to_post(recent_posts_cache: {}, message_json: {},
base_dir: str, base_dir: str,
nickname: str, domain: str, nickname: str, domain: str,
max_mentions: int, max_emoji: int, max_mentions: int, max_emoji: int,
allow_local_network_access: bool, allow_local_network_access: bool,
debug: bool, debug: bool,
system_language: str, http_prefix: str, system_language: str, http_prefix: str,
domain_full: str, person_cache: {}, domain_full: str, person_cache: {},
signing_priv_key_pem: str, signing_priv_key_pem: str,
max_recent_posts: int, translate: {}, max_recent_posts: int, translate: {},
session, cached_webfingers: {}, port: int, session, cached_webfingers: {}, port: int,
allow_deletion: bool, allow_deletion: bool,
yt_replace_domain: str, yt_replace_domain: str,
twitter_replacement_domain: str, twitter_replacement_domain: str,
show_published_date_only: bool, show_published_date_only: bool,
peertube_instances: [], peertube_instances: [],
theme_name: str, max_like_count: int, theme_name: str, max_like_count: int,
cw_lists: {}, dogwhistles: {}, cw_lists: {}, dogwhistles: {},
min_images_for_accounts: []) -> bool: min_images_for_accounts: []) -> bool:
"""A post was edited """A post was edited
""" """
if not has_object_dict(message_json): if not has_object_dict(message_json):
@ -1205,7 +1205,7 @@ def _receive_edit_to_post(recent_posts_cache: {}, message_json: {},
# has the content changed? # has the content changed?
if post_json_object['object']['content'] == \ if post_json_object['object']['content'] == \
message_json['object']['content']: message_json['object']['content']:
# same content # same content. Has the summary changed?
if 'summary' in post_json_object['object'] and \ if 'summary' in post_json_object['object'] and \
'summary' in message_json['object']: 'summary' in message_json['object']:
if post_json_object['object']['summary'] == \ if post_json_object['object']['summary'] == \
@ -1329,23 +1329,23 @@ def _receive_update_activity(recent_posts_cache: {}, session, base_dir: str,
elif message_json['object']['type'] == 'Note': elif message_json['object']['type'] == 'Note':
if message_json['object'].get('id'): if message_json['object'].get('id'):
domain_full = get_full_domain(domain, port) domain_full = get_full_domain(domain, port)
if _receive_edit_to_post(recent_posts_cache, message_json, if receive_edit_to_post(recent_posts_cache, message_json,
base_dir, nickname, domain, base_dir, nickname, domain,
max_mentions, max_emoji, max_mentions, max_emoji,
allow_local_network_access, allow_local_network_access,
debug, system_language, http_prefix, debug, system_language, http_prefix,
domain_full, person_cache, domain_full, person_cache,
signing_priv_key_pem, signing_priv_key_pem,
max_recent_posts, translate, max_recent_posts, translate,
session, cached_webfingers, port, session, cached_webfingers, port,
allow_deletion, allow_deletion,
yt_replace_domain, yt_replace_domain,
twitter_replacement_domain, twitter_replacement_domain,
show_published_date_only, show_published_date_only,
peertube_instances, peertube_instances,
theme_name, max_like_count, theme_name, max_like_count,
cw_lists, dogwhistles, cw_lists, dogwhistles,
min_images_for_accounts): min_images_for_accounts):
print('EDITPOST: received ' + message_json['object']['id']) print('EDITPOST: received ' + message_json['object']['id'])
return True return True
else: else:

265
outbox.py
View File

@ -414,148 +414,157 @@ def post_message_to_outbox(session, translate: {},
post_id = None post_id = None
if debug: if debug:
print('DEBUG: save_post_to_box') print('DEBUG: save_post_to_box')
if message_json['type'] != 'Upgrade':
outbox_name = 'outbox'
store_hash_tags(base_dir, post_to_nickname, domain, is_edited_post = False
http_prefix, domain_full, if message_json['type'] == 'Update' and \
message_json, translate) message_json['object']['type'] == 'Note':
is_edited_post = True
message_json['type'] = 'Create'
# if this is a blog post or an event then save to its own box outbox_name = 'outbox'
if message_json['type'] == 'Create':
if has_object_dict(message_json):
if message_json['object'].get('type'):
if message_json['object']['type'] == 'Article':
outbox_name = 'tlblogs'
saved_filename = \ store_hash_tags(base_dir, post_to_nickname, domain,
save_post_to_box(base_dir, http_prefix, domain_full,
http_prefix, message_json, translate)
post_id,
post_to_nickname, domain_full,
message_json, outbox_name)
if not saved_filename:
print('WARN: post not saved to outbox ' + outbox_name)
return False
update_speaker(base_dir, http_prefix, # if this is a blog post or an event then save to its own box
post_to_nickname, domain, domain_full, if message_json['type'] == 'Create':
message_json, person_cache, if has_object_dict(message_json):
translate, message_json['actor'], if message_json['object'].get('type'):
theme, system_language, if message_json['object']['type'] == 'Article':
outbox_name) outbox_name = 'tlblogs'
# save all instance blogs to the news actor saved_filename = \
if post_to_nickname != 'news' and outbox_name == 'tlblogs': save_post_to_box(base_dir,
if '/' in saved_filename: http_prefix,
if is_featured_writer(base_dir, post_to_nickname, domain): post_id,
saved_post_id = saved_filename.split('/')[-1] post_to_nickname, domain_full,
blogs_dir = \ message_json, outbox_name)
base_dir + '/accounts/news@' + domain + '/tlblogs' if not saved_filename:
if not os.path.isdir(blogs_dir): print('WARN: post not saved to outbox ' + outbox_name)
os.mkdir(blogs_dir) return False
copyfile(saved_filename, blogs_dir + '/' + saved_post_id)
inbox_update_index('tlblogs', base_dir,
'news@' + domain,
saved_filename, debug)
# clear the citations file if it exists update_speaker(base_dir, http_prefix,
citations_filename = \ post_to_nickname, domain, domain_full,
base_dir + '/accounts/' + \ message_json, person_cache,
post_to_nickname + '@' + domain + '/.citations.txt' translate, message_json['actor'],
if os.path.isfile(citations_filename): theme, system_language,
try: outbox_name)
os.remove(citations_filename)
except OSError:
print('EX: post_message_to_outbox unable to delete ' +
citations_filename)
# The following activity types get added to the index files # save all instance blogs to the news actor
indexed_activities = ( if post_to_nickname != 'news' and outbox_name == 'tlblogs':
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article', if '/' in saved_filename:
'Patch', 'Announce', 'ChatMessage' if is_featured_writer(base_dir, post_to_nickname, domain):
) saved_post_id = saved_filename.split('/')[-1]
if message_json['type'] in indexed_activities: blogs_dir = \
indexes = [outbox_name, "inbox"] base_dir + '/accounts/news@' + domain + '/tlblogs'
self_actor = \ if not os.path.isdir(blogs_dir):
local_actor_url(http_prefix, post_to_nickname, domain_full) os.mkdir(blogs_dir)
for box_name_index in indexes: copyfile(saved_filename, blogs_dir + '/' + saved_post_id)
if not box_name_index: inbox_update_index('tlblogs', base_dir,
continue 'news@' + domain,
saved_filename, debug)
# should this also go to the media timeline? # clear the citations file if it exists
if box_name_index == 'inbox': citations_filename = \
if is_image_media(session, base_dir, http_prefix, base_dir + '/accounts/' + \
post_to_nickname, domain, post_to_nickname + '@' + domain + '/.citations.txt'
message_json, if os.path.isfile(citations_filename):
yt_replace_domain, try:
twitter_replacement_domain, os.remove(citations_filename)
allow_local_network_access, except OSError:
recent_posts_cache, debug, print('EX: post_message_to_outbox unable to delete ' +
system_language, citations_filename)
domain_full, person_cache,
signing_priv_key_pem,
bold_reading):
inbox_update_index('tlmedia', base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
if box_name_index == 'inbox' and outbox_name == 'tlblogs': # The following activity types get added to the index files
continue indexed_activities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce', 'ChatMessage'
)
if message_json['type'] in indexed_activities:
indexes = [outbox_name, "inbox"]
self_actor = \
local_actor_url(http_prefix, post_to_nickname, domain_full)
for box_name_index in indexes:
if not box_name_index:
continue
# avoid duplicates of the message if already going # should this also go to the media timeline?
# back to the inbox of the same account if box_name_index == 'inbox':
if self_actor not in message_json['to']: if is_image_media(session, base_dir, http_prefix,
# show sent post within the inbox, post_to_nickname, domain,
# as is the typical convention message_json,
inbox_update_index(box_name_index, base_dir, yt_replace_domain,
twitter_replacement_domain,
allow_local_network_access,
recent_posts_cache, debug,
system_language,
domain_full, person_cache,
signing_priv_key_pem,
bold_reading):
inbox_update_index('tlmedia', base_dir,
post_to_nickname + '@' + domain, post_to_nickname + '@' + domain,
saved_filename, debug) saved_filename, debug)
# regenerate the html if box_name_index == 'inbox' and outbox_name == 'tlblogs':
use_cache_only = False continue
page_number = 1
show_individual_post_icons = True # avoid duplicates of the message if already going
manually_approve_followers = \ # back to the inbox of the same account
follower_approval_active(base_dir, if self_actor not in message_json['to']:
post_to_nickname, domain) # show sent post within the inbox,
timezone = \ # as is the typical convention
get_account_timezone(base_dir, inbox_update_index(box_name_index, base_dir,
post_to_nickname + '@' + domain,
saved_filename, debug)
# regenerate the html
use_cache_only = False
page_number = 1
show_individual_post_icons = True
manually_approve_followers = \
follower_approval_active(base_dir,
post_to_nickname, domain) post_to_nickname, domain)
mitm = False timezone = \
if os.path.isfile(saved_filename.replace('.json', '') + get_account_timezone(base_dir,
'.mitm'): post_to_nickname, domain)
mitm = True mitm = False
minimize_all_images = False if os.path.isfile(saved_filename.replace('.json', '') +
if post_to_nickname in min_images_for_accounts: '.mitm'):
minimize_all_images = True mitm = True
individual_post_as_html(signing_priv_key_pem, minimize_all_images = False
False, recent_posts_cache, if post_to_nickname in min_images_for_accounts:
max_recent_posts, minimize_all_images = True
translate, page_number, individual_post_as_html(signing_priv_key_pem,
base_dir, session, False, recent_posts_cache,
cached_webfingers, max_recent_posts,
person_cache, translate, page_number,
post_to_nickname, domain, port, base_dir, session,
message_json, None, True, cached_webfingers,
allow_deletion, person_cache,
http_prefix, __version__, post_to_nickname, domain, port,
box_name_index, message_json, None, True,
yt_replace_domain, allow_deletion,
twitter_replacement_domain, http_prefix, __version__,
show_published_date_only, box_name_index,
peertube_instances, yt_replace_domain,
allow_local_network_access, twitter_replacement_domain,
theme, system_language, show_published_date_only,
max_like_count, peertube_instances,
box_name_index != 'dm', allow_local_network_access,
show_individual_post_icons, theme, system_language,
manually_approve_followers, max_like_count,
False, True, use_cache_only, box_name_index != 'dm',
cw_lists, lists_enabled, show_individual_post_icons,
timezone, mitm, manually_approve_followers,
bold_reading, dogwhistles, False, True, use_cache_only,
minimize_all_images) cw_lists, lists_enabled,
timezone, mitm,
bold_reading, dogwhistles,
minimize_all_images)
if is_edited_post:
message_json['type'] = 'Update'
if outbox_announce(recent_posts_cache, if outbox_announce(recent_posts_cache,
base_dir, message_json, debug): base_dir, message_json, debug):