indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Remove rejected posts from inbox index

merge-requests/30/head
Bob Mottram 2024-10-23 13:20:07 +01:00
parent b9b3ad6c0f
commit ca995cd50d
3 changed files with 67 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
posts.py
View File

@ -41,6 +41,7 @@ from flags import contains_private_key
from flags import has_group_type from flags import has_group_type
from flags import is_premium_account from flags import is_premium_account
from flags import url_permitted from flags import url_permitted
from utils import remove_post_from_index
from utils import replace_strings from utils import replace_strings
from utils import valid_content_warning from utils import valid_content_warning
from utils import get_actor_from_post_id from utils import get_actor_from_post_id
@ -4884,6 +4885,7 @@ def _create_box_items(base_dir: str,
print('REJECT: rejected post in timeline ' + print('REJECT: rejected post in timeline ' +
boxname + ' ' + post_url + ' ' + boxname + ' ' + post_url + ' ' +
full_post_filename) full_post_filename)
remove_post_from_index(post_url, False, index_filename)
continue continue
if _add_post_to_timeline(full_post_filename, boxname, if _add_post_to_timeline(full_post_filename, boxname,
@ -6150,11 +6152,12 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str,
def _reject_announce(announce_filename: str, def _reject_announce(announce_filename: str,
base_dir: str, nickname: str, domain: str, base_dir: str, nickname: str, domain: str,
announce_post_id: str, recent_posts_cache: {}): announce_post_id: str, recent_posts_cache: {},
debug: bool):
"""Marks an announce as rejected """Marks an announce as rejected
""" """
reject_post_id(base_dir, nickname, domain, announce_post_id, reject_post_id(base_dir, nickname, domain, announce_post_id,
recent_posts_cache) recent_posts_cache, debug)
# reject the post referenced by the announce activity object # reject the post referenced by the announce activity object
if not os.path.isfile(announce_filename + '.reject'): if not os.path.isfile(announce_filename + '.reject'):
@ -6282,7 +6285,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if announced_json.get('error'): if announced_json.get('error'):
print('WARN: ' + print('WARN: ' +
@ -6295,7 +6298,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
announced_actor = announced_json['id'] announced_actor = announced_json['id']
@ -6307,7 +6310,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if announced_json['type'] == 'Video': if announced_json['type'] == 'Video':
converted_json = \ converted_json = \
@ -6323,14 +6326,14 @@ def download_announce(session, base_dir: str, http_prefix: str,
'or /objects/ or /p/ ' + str(announced_json)) 'or /objects/ or /p/ ' + str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if not has_users_path(announced_actor): if not has_users_path(announced_actor):
print('WARN: announced post id does not contain /users/ ' + print('WARN: announced post id does not contain /users/ ' +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if announced_json['type'] not in ('Note', 'Event', 'Page', if announced_json['type'] not in ('Note', 'Event', 'Page',
'Question', 'Article'): 'Question', 'Article'):
@ -6339,7 +6342,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
# You can only announce Note, Page, Article or Question types # You can only announce Note, Page, Article or Question types
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if announced_json['type'] == 'Question': if announced_json['type'] == 'Question':
if not announced_json.get('endTime') or \ if not announced_json.get('endTime') or \
@ -6350,14 +6353,14 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if not isinstance(announced_json['oneOf'], list): if not isinstance(announced_json['oneOf'], list):
print('WARN: announced Question oneOf should be a list ' + print('WARN: announced Question oneOf should be a list ' +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if is_question_filtered(base_dir, nickname, domain, if is_question_filtered(base_dir, nickname, domain,
system_language, announced_json): system_language, announced_json):
@ -6366,21 +6369,21 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json['id'])) str(announced_json['id']))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if 'content' not in announced_json: if 'content' not in announced_json:
print('WARN: announced post does not have content ' + print('WARN: announced post does not have content ' +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if not announced_json.get('published'): if not announced_json.get('published'):
print('WARN: announced post does not have published ' + print('WARN: announced post does not have published ' +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if '.' in announced_json['published'] and \ if '.' in announced_json['published'] and \
'Z' in announced_json['published']: 'Z' in announced_json['published']:
@ -6391,7 +6394,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json['published'])) str(announced_json['published']))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if not understood_post_language(base_dir, nickname, if not understood_post_language(base_dir, nickname,
announced_json, system_language, announced_json, system_language,
@ -6411,7 +6414,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
summary_str = \ summary_str = \
get_summary_from_post(announced_json, system_language, []) get_summary_from_post(announced_json, system_language, [])
@ -6428,7 +6431,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json['id'])) str(announced_json['id']))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if reject_twitter_summary(base_dir, nickname, domain, if reject_twitter_summary(base_dir, nickname, domain,
summary_str): summary_str):
@ -6436,21 +6439,21 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if contains_private_key(content_str): if contains_private_key(content_str):
print("WARN: announced post contains someone's private key " + print("WARN: announced post contains someone's private key " +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
if invalid_ciphertext(content_str): if invalid_ciphertext(content_str):
print('WARN: announced post contains invalid ciphertext ' + print('WARN: announced post contains invalid ciphertext ' +
str(announced_json)) str(announced_json))
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
# remove any long words # remove any long words
@ -6478,7 +6481,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
# Create wrap failed # Create wrap failed
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
# if poll/vote/question is not to be shown # if poll/vote/question is not to be shown
@ -6504,7 +6507,7 @@ def download_announce(session, base_dir: str, http_prefix: str,
str(attributed_nickname) + '@' + attributed_domain) str(attributed_nickname) + '@' + attributed_domain)
_reject_announce(announce_filename, _reject_announce(announce_filename,
base_dir, nickname, domain, post_id, base_dir, nickname, domain, post_id,
recent_posts_cache) recent_posts_cache, debug)
return None return None
post_json_object = announced_json post_json_object = announced_json
replace_you_tube(post_json_object, yt_replace_domain, system_language) replace_you_tube(post_json_object, yt_replace_domain, system_language)

42
utils.py
View File

@ -2233,26 +2233,26 @@ def _remove_attachment(base_dir: str, http_prefix: str, domain: str,
post_json['attachment'] = [] post_json['attachment'] = []
def remove_moderation_post_from_index(base_dir: str, post_url: str, def remove_post_from_index(post_url: str, debug: bool,
debug: bool) -> None: index_file: str) -> None:
"""Removes a url from the moderation index """Removes a url from a box index
""" """
moderation_index_file = data_dir(base_dir) + '/moderation.txt' if not os.path.isfile(index_file):
if not os.path.isfile(moderation_index_file):
return return
post_id = remove_id_ending(post_url) post_id = remove_id_ending(post_url)
if text_in_file(post_id, moderation_index_file): if not text_in_file(post_id, index_file):
return
lines = [] lines = []
try: try:
with open(moderation_index_file, 'r', encoding='utf-8') as fp_mod1: with open(index_file, 'r', encoding='utf-8') as fp_mod1:
lines = fp_mod1.readlines() lines = fp_mod1.readlines()
except OSError as exc: except OSError as exc:
print('EX: remove_moderation_post_from_index unable to read ' + print('EX: remove_post_from_index unable to read ' +
moderation_index_file + ' ' + str(exc)) index_file + ' ' + str(exc))
if lines: if lines:
try: try:
with open(moderation_index_file, 'w+', with open(index_file, 'w+',
encoding='utf-8') as fp_mod2: encoding='utf-8') as fp_mod2:
for line in lines: for line in lines:
if line.strip("\n").strip("\r") != post_id: if line.strip("\n").strip("\r") != post_id:
@ -2260,11 +2260,19 @@ def remove_moderation_post_from_index(base_dir: str, post_url: str,
continue continue
if debug: if debug:
print('DEBUG: removed ' + post_id + print('DEBUG: removed ' + post_id +
' from moderation index') ' from index ' + index_file)
except OSError as exc: except OSError as exc:
print('EX: ' + print('EX: ' +
'remove_moderation_post_from_index unable to write ' + 'remove_post_from_index unable to write ' +
moderation_index_file + ' ' + str(exc)) index_file + ' ' + str(exc))
def remove_moderation_post_from_index(base_dir: str, post_url: str,
debug: bool) -> None:
"""Removes a url from the moderation index
"""
moderation_index_file = data_dir(base_dir) + '/moderation.txt'
remove_post_from_index(post_url, debug, moderation_index_file)
def _is_reply_to_blog_post(base_dir: str, nickname: str, domain: str, def _is_reply_to_blog_post(base_dir: str, nickname: str, domain: str,
@ -3536,7 +3544,8 @@ def _convert_to_camel_case(text: str) -> str:
def reject_post_id(base_dir: str, nickname: str, domain: str, def reject_post_id(base_dir: str, nickname: str, domain: str,
post_id: str, recent_posts_cache: {}) -> None: post_id: str, recent_posts_cache: {},
debug: bool) -> None:
""" Marks the given post as rejected, """ Marks the given post as rejected,
for example an announce which is too old for example an announce which is too old
""" """
@ -3570,6 +3579,11 @@ def reject_post_id(base_dir: str, nickname: str, domain: str,
print('EX: reject_post_id unable to write ' + print('EX: reject_post_id unable to write ' +
post_filename + '.reject') post_filename + '.reject')
# if the post is in the inbox index then remove it
index_file = \
acct_dir(base_dir, nickname, domain) + '/inbox.index'
remove_post_from_index(post_url, debug, index_file)
def load_translations_from_file(base_dir: str, language: str) -> ({}, str): def load_translations_from_file(base_dir: str, language: str) -> ({}, str):
"""Returns the translations dictionary """Returns the translations dictionary

2
webapp_post.py
View File

@ -2330,7 +2330,7 @@ def individual_post_as_html(signing_priv_key_pem: str,
# if the announce could not be downloaded then mark it as rejected # if the announce could not be downloaded then mark it as rejected
announced_post_id = remove_id_ending(post_json_object['id']) announced_post_id = remove_id_ending(post_json_object['id'])
reject_post_id(base_dir, nickname, domain, announced_post_id, reject_post_id(base_dir, nickname, domain, announced_post_id,
recent_posts_cache) recent_posts_cache, False)
return '' return ''
post_json_object = post_json_announce post_json_object = post_json_announce