indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Merge branch 'main' of gitlab.com:bashrc2/epicyon

merge-requests/30/head
Bob Mottram 2022-04-11 13:27:05 +01:00
parent c168cb436d d7bb5dc1c7
commit ab2e1dcf67
27 changed files with 308 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
content.py
View File

@ -7,12 +7,16 @@ __email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Core"
import difflib
import math
import html
import os
import email.parser
import urllib.parse
from shutil import copyfile
from dateutil.parser import parse
from utils import convert_published_to_local_timezone
from utils import has_object_dict
from utils import valid_hash_tag
from utils import dangerous_svg
from utils import remove_domain_port
@ -1415,3 +1419,87 @@ def import_emoji(base_dir: str, import_filename: str, session) -> None:
added += 1
save_json(emoji_dict, base_dir + '/emoji/default_emoji.json')
print(str(added) + ' custom emoji added')
def content_diff(content: str, prev_content: str) -> str:
"""Returns a diff for the given content
"""
d = difflib.Differ()
text1_lines = content.splitlines()
text1_sentences = []
for line in text1_lines:
sentences = line.split('.')
for sentence in sentences:
text1_sentences.append(sentence.strip())
text2_lines = prev_content.splitlines()
text2_sentences = []
for line in text2_lines:
sentences = line.split('.')
for sentence in sentences:
text2_sentences.append(sentence.strip())
diff = d.compare(text1_sentences, text2_sentences)
diff_text = ''
for line in diff:
if line.startswith('- '):
if not diff_text:
diff_text = '<p>'
else:
diff_text += '<br>'
diff_text += '<label class="diff_add">+ ' + line[2:] + '</label>'
elif line.startswith('+ '):
if not diff_text:
diff_text = '<p>'
else:
diff_text += '<br>'
diff_text += \
'<label class="diff_remove">- ' + line[2:] + '</label>'
if diff_text:
diff_text += '</p>'
return diff_text
def create_edits_html(edits_json: {}, post_json_object: {},
translate: {}, timezone: str) -> str:
""" Creates html showing historical edits made to a post
"""
if not edits_json:
return ''
if not has_object_dict(post_json_object):
return ''
if not post_json_object['object'].get('content'):
return ''
edit_dates_list = []
for modified, item in edits_json.items():
edit_dates_list.append(modified)
edit_dates_list.sort(reverse=True)
edits_str = ''
content = remove_html(post_json_object['object']['content'])
for modified in edit_dates_list:
prev_json = edits_json[modified]
if not has_object_dict(prev_json):
continue
if not prev_json['object'].get('content'):
continue
prev_content = remove_html(prev_json['object']['content'])
if content == prev_content:
continue
diff = content_diff(content, prev_content)
if not diff:
continue
diff = diff.replace('\n', '</p><p>')
# convert to local time
datetime_object = parse(modified)
datetime_object = \
convert_published_to_local_timezone(datetime_object, timezone)
modified_str = datetime_object.strftime("%a %b %d, %H:%M")
diff = '<p><b>' + modified_str + '</b></p>' + diff
edits_str += diff
content = prev_content
if not edits_str:
return ''
return '<br><details><summary class="cw">' + \
translate['SHOW EDITS'] + '</summary>' + \
edits_str + '</details>'

2
daemon.py
View File

@ -20792,7 +20792,7 @@ def run_daemon(check_actor_timeout: int,
check_actor_timeout = 2
httpd.check_actor_timeout = check_actor_timeout
# how many hours after a post was publushed can a reply be made
# how many hours after a post was published can a reply be made
default_reply_interval_hrs = 9999999
httpd.default_reply_interval_hrs = default_reply_interval_hrs

10
epicyon-profile.css
View File

@ -262,6 +262,16 @@ mark {
font-weight: bold;
}
.diff_add {
color: var(--main-link-color);
text-decoration: underline;
}
.diff_remove {
color: var(--title-color);
text-decoration: underline;
}
.accesskeys {
border: 0;
width: 100%;

63
inbox.py
View File

@ -12,6 +12,7 @@ import os
import datetime
import time
import random
from shutil import copyfile
from linked_data_sig import verify_json_signature
from languages import understood_post_language
from like import update_likes_collection
@ -1064,8 +1065,9 @@ def _receive_edit_to_post(recent_posts_cache: {}, message_json: {},
published_str = post_json_object['object']['updated']
else:
published_str = post_json_object['object']['published']
post_history_json[published_str] = post_json_object
save_json(post_history_json, post_history_filename)
if not post_history_json.get(published_str):
post_history_json[published_str] = post_json_object
save_json(post_history_json, post_history_filename)
# Change Update to Create
message_json['type'] = 'Create'
save_json(message_json, post_filename)
@ -3930,6 +3932,50 @@ def _inbox_after_initial(server,
except OSError:
print('EX: unable to write ' + destination_filename_muted)
# is this an edit of a previous post?
# in Mastodon "delete and redraft"
# NOTE: this must be done before update_conversation is called
edited_filename, edited_json = \
edited_post_filename(base_dir, handle_name, domain,
post_json_object, debug, 300)
# If this was an edit then update the edits json file and
# delete the previous version of the post
if edited_filename and edited_json:
prev_edits_filename = \
edited_filename.replace('.json', '.edits')
edits_filename = \
destination_filename.replace('.json', '.edits')
modified = edited_json['object']['published']
if os.path.isfile(edits_filename):
edits_json = load_json(edits_filename)
if edits_json:
if not edits_json.get(modified):
edits_json[modified] = edited_json
save_json(edits_json, edits_filename)
else:
if os.path.isfile(prev_edits_filename):
if prev_edits_filename != edits_filename:
try:
copyfile(prev_edits_filename, edits_filename)
except OSError:
print('EX: failed to copy edits file')
edits_json = load_json(edits_filename)
if edits_json:
if not edits_json.get(modified):
edits_json[modified] = edited_json
save_json(edits_json, edits_filename)
else:
edits_json = {
modified: edited_json
}
save_json(edits_json, edits_filename)
if edited_filename != destination_filename:
delete_post(base_dir, http_prefix,
nickname, domain, edited_filename,
debug, recent_posts_cache)
# update the indexes for different timelines
for boxname in update_index_list:
if not inbox_update_index(boxname, base_dir, handle,
@ -3983,22 +4029,9 @@ def _inbox_after_initial(server,
boxname + ' post as html to cache in ' +
time_diff + ' mS')
# is this an edit of a previous post?
# in Mastodon "delete and redraft"
# NOTE: this must be done before update_conversation is called
edited_filename = \
edited_post_filename(base_dir, handle_name, domain,
post_json_object, debug, 300)
update_conversation(base_dir, handle_name, domain,
post_json_object)
# If this was an edit then delete the previous version of the post
if edited_filename:
delete_post(base_dir, http_prefix,
nickname, domain, edited_filename,
debug, recent_posts_cache)
# store the id of the last post made by this actor
_store_last_post_id(base_dir, nickname, domain, post_json_object)

54
posts.py
View File

@ -5476,31 +5476,31 @@ def seconds_between_published(published1: str, published2: str) -> int:
def edited_post_filename(base_dir: str, nickname: str, domain: str,
post_json_object: {}, debug: bool,
max_time_diff_seconds: int) -> str:
max_time_diff_seconds: int) -> (str, {}):
"""Returns the filename of the edited post
"""
if not has_object_dict(post_json_object):
return ''
return '', None
if not post_json_object.get('type'):
return ''
return '', None
if not post_json_object['object'].get('type'):
return ''
return '', None
if not post_json_object['object'].get('published'):
return ''
return '', None
if not post_json_object['object'].get('id'):
return ''
return '', None
if not post_json_object['object'].get('content'):
return ''
return '', None
if not post_json_object['object'].get('attributedTo'):
return ''
return '', None
if not isinstance(post_json_object['object']['attributedTo'], str):
return ''
return '', None
actor = post_json_object['object']['attributedTo']
actor_filename = \
acct_dir(base_dir, nickname, domain) + '/lastpost/' + \
actor.replace('/', '#')
if not os.path.isfile(actor_filename):
return ''
return '', None
post_id = remove_id_ending(post_json_object['object']['id'])
lastpost_id = None
try:
@ -5508,48 +5508,48 @@ def edited_post_filename(base_dir: str, nickname: str, domain: str,
lastpost_id = fp_actor.read()
except OSError:
print('EX: edited_post_filename unable to read ' + actor_filename)
return ''
return '', None
if not lastpost_id:
return ''
return '', None
if lastpost_id == post_id:
return ''
return '', None
lastpost_filename = \
locate_post(base_dir, nickname, domain, lastpost_id, False)
if not lastpost_filename:
return ''
return '', None
lastpost_json = load_json(lastpost_filename, 0)
if not lastpost_json:
return ''
return '', None
if not lastpost_json.get('type'):
return ''
return '', None
if lastpost_json['type'] != post_json_object['type']:
return ''
return '', None
if not lastpost_json['object'].get('type'):
return ''
return '', None
if lastpost_json['object']['type'] != post_json_object['object']['type']:
return
return '', None
if not lastpost_json['object'].get('published'):
return ''
return '', None
if not lastpost_json['object'].get('id'):
return ''
return '', None
if not lastpost_json['object'].get('content'):
return ''
return '', None
if not lastpost_json['object'].get('attributedTo'):
return ''
return '', None
if not isinstance(lastpost_json['object']['attributedTo'], str):
return ''
return '', None
time_diff_seconds = \
seconds_between_published(lastpost_json['object']['published'],
post_json_object['object']['published'])
if time_diff_seconds > max_time_diff_seconds:
return ''
return '', None
if debug:
print(post_id + ' might be an edit of ' + lastpost_id)
if words_similarity(lastpost_json['object']['content'],
post_json_object['object']['content'], 10) < 70:
return ''
return '', None
print(post_id + ' is an edit of ' + lastpost_id)
return lastpost_filename
return lastpost_filename, lastpost_json
def get_original_post_from_announce_url(announce_url: str, base_dir: str,

79
tests.py
View File

@ -129,6 +129,8 @@ from inbox import json_post_allows_comments
from inbox import valid_inbox
from inbox import valid_inbox_filenames
from categories import guess_hashtag_category
from content import create_edits_html
from content import content_diff
from content import bold_reading_string
from content import safe_web_text
from content import words_similarity
@ -6917,6 +6919,82 @@ def _test_bold_reading() -> None:
assert text_bold == expected
def _test_diff_content() -> None:
print('diff_content')
prev_content = \
'Some text before.\n' + \
'Starting sentence. This is some content.\nThis is another line.'
content = \
'Some text before.\nThis is some more content.\nThis is another line.'
result = content_diff(content, prev_content)
expected = \
'<p><label class="diff_remove">' + \
'- Starting sentence</label><br><label class="diff_add">' + \
'+ This is some more content</label><br>' + \
'<label class="diff_remove">- This is some content</label></p>'
assert result == expected
content = \
'Some text before.\nThis is content.\nThis line.'
result = content_diff(content, prev_content)
expected = \
'<p><label class="diff_remove">- Starting sentence</label><br>' + \
'<label class="diff_add">+ This is content</label><br>' + \
'<label class="diff_remove">- This is some content</label><br>' + \
'<label class="diff_add">+ This line</label><br>' + \
'<label class="diff_remove">- This is another line</label></p>'
assert result == expected
translate = {
"SHOW EDITS": "SHOW EDITS"
}
timezone = 'Europe/Berlin'
content1 = \
"<p>This is some content.</p>" + \
"<p>Some other content.</p>"
content2 = \
"<p>This is some previous content.</p>" + \
"<p>Some other previous content.</p>"
content3 = \
"<p>This is some more previous content.</p>" + \
"<p>Some other previous content.</p>"
post_json_object = {
"object": {
"content": content1,
"published": "2020-12-14T00:08:06Z"
}
}
edits_json = {
"2020-12-14T00:05:19Z": {
"object": {
"content": content3,
"published": "2020-12-14T00:05:19Z"
}
},
"2020-12-14T00:07:34Z": {
"object": {
"content": content2,
"published": "2020-12-14T00:07:34Z"
}
}
}
html_str = \
create_edits_html(edits_json, post_json_object, translate, timezone)
assert html_str
expected = \
'<br><details><summary class="cw">SHOW EDITS</summary>' + \
'<p><b>Mon Dec 14, 01:07</b></p><p><label class="diff_add">' + \
'+ This is some content</label><br><label class="diff_remove">' + \
'- This is some previous content</label><br>' + \
'<label class="diff_add">+ Some other content</label><br>' + \
'<label class="diff_remove">- Some other previous content' + \
'</label></p><p><b>Mon Dec 14, 01:05</b></p><p>' + \
'<label class="diff_add">+ This is some previous content' + \
'</label><br><label class="diff_remove">' + \
'- This is some more previous content</label></p></details>'
assert html_str == expected
def run_all_tests():
base_dir = os.getcwd()
print('Running tests...')
@ -6934,6 +7012,7 @@ def run_all_tests():
_test_checkbox_names()
_test_thread_functions()
_test_functions()
_test_diff_content()
_test_bold_reading()
_test_published_to_local_timezone()
_test_safe_webtext()

3
translations/ar.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "مسموح روبوتات الويب",
"Known Search Bots": "روبوتات بحث الويب المعروفة",
"mitm": "يمكن قراءة الرسالة أو تعديلها من قبل طرف ثالث",
"Bold reading": "قراءة جريئة"
"Bold reading": "قراءة جريئة",
"SHOW EDITS": "عرض التعديلات"
}

3
translations/ca.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Bots web permesos",
"Known Search Bots": "Bots de cerca web coneguts",
"mitm": "El missatge podria haver estat llegit o modificat per un tercer",
"Bold reading": "Lectura atrevida"
"Bold reading": "Lectura atrevida",
"SHOW EDITS": "MOSTRA EDICIONS"
}

3
translations/cy.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Web Bots a Ganiateir",
"Known Search Bots": "Bots Chwilio Gwe Hysbys",
"mitm": "Gallai'r neges fod wedi cael ei darllen neu ei haddasu gan drydydd parti",
"Bold reading": "Darllen beiddgar"
"Bold reading": "Darllen beiddgar",
"SHOW EDITS": "GOLYGIADAU SIOE"
}

3
translations/de.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Webbots erlaubt",
"Known Search Bots": "Bekannte Bots für die Websuche",
"mitm": "Die Nachricht könnte von einem Dritten gelesen oder geändert worden sein",
"Bold reading": "Mutige Lektüre"
"Bold reading": "Mutige Lektüre",
"SHOW EDITS": "BEARBEITUNGEN ZEIGEN"
}

3
translations/en.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Web Search Bots Allowed",
"Known Search Bots": "Known Web Search Bots",
"mitm": "Message could have been read or modified by a third party",
"Bold reading": "Bold reading"
"Bold reading": "Bold reading",
"SHOW EDITS": "SHOW EDITS"
}

3
translations/es.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Bots web permitidos",
"Known Search Bots": "Bots de búsqueda web conocidos",
"mitm": "El mensaje podría haber sido leído o modificado por un tercero",
"Bold reading": "Lectura en negrita"
"Bold reading": "Lectura en negrita",
"SHOW EDITS": "MOSTRAR EDICIONES"
}

3
translations/fr.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Robots Web autorisés",
"Known Search Bots": "Robots de recherche Web connus",
"mitm": "Le message a pu être lu ou modifié par un tiers",
"Bold reading": "Lecture audacieuse"
"Bold reading": "Lecture audacieuse",
"SHOW EDITS": "AFFICHER LES MODIFICATIONS"
}

3
translations/ga.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Róbónna Gréasáin Ceadaithe",
"Known Search Bots": "Róbónna Cuardach Gréasáin Aitheanta",
"mitm": "D'fhéadfadh tríú páirtí an teachtaireacht a léamh nó a mhodhnú",
"Bold reading": "Léamh trom"
"Bold reading": "Léamh trom",
"SHOW EDITS": "EAGARTHÓIRÍ TAISPEÁINT"
}

3
translations/hi.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "वेब बॉट्स की अनुमति है",
"Known Search Bots": "ज्ञात वेब खोज बॉट्स",
"mitm": "संदेश किसी तीसरे पक्ष द्वारा पढ़ा या संशोधित किया जा सकता था",
"Bold reading": "बोल्ड रीडिंग"
"Bold reading": "बोल्ड रीडिंग",
"SHOW EDITS": "संपादन दिखाएं"
}

3
translations/it.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Web bot consentiti",
"Known Search Bots": "Bot di ricerca Web noti",
"mitm": "Il messaggio potrebbe essere stato letto o modificato da terzi",
"Bold reading": "Lettura audace"
"Bold reading": "Lettura audace",
"SHOW EDITS": "MOSTRA MODIFICHE"
}

3
translations/ja.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "許可されたWebボット",
"Known Search Bots": "既知のWeb検索ボット",
"mitm": "メッセージが第三者によって読み取られたり変更されたりした可能性があります",
"Bold reading": "大胆な読書"
"Bold reading": "大胆な読書",
"SHOW EDITS": "編集を表示"
}

3
translations/ko.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "웹 봇 허용",
"Known Search Bots": "알려진 웹 검색 봇",
"mitm": "제3자가 메시지를 읽거나 수정했을 수 있습니다.",
"Bold reading": "굵은 글씨"
"Bold reading": "굵은 글씨",
"SHOW EDITS": "수정사항 보기"
}

3
translations/ku.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Web Bots Destûrdar in",
"Known Search Bots": "Botên Lêgerîna Webê yên naskirî",
"mitm": "Peyam dikaribû ji hêla aliyek sêyemîn ve were xwendin an guhertin",
"Bold reading": "Xwendina qelew"
"Bold reading": "Xwendina qelew",
"SHOW EDITS": "GERÎŞTAN NÎŞAN DE"
}

3
translations/oc.json
View File

@ -515,5 +515,6 @@
"Web Bots Allowed": "Web Search Bots Allowed",
"Known Search Bots": "Known Web Search Bots",
"mitm": "Message could have been read or modified by a third party",
"Bold reading": "Bold reading"
"Bold reading": "Bold reading",
"SHOW EDITS": "SHOW EDITS"
}

3
translations/pl.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Dozwolone boty internetowe",
"Known Search Bots": "Znane boty wyszukiwania w sieci",
"mitm": "Wiadomość mogła zostać przeczytana lub zmodyfikowana przez osobę trzecią",
"Bold reading": "Odważne czytanie"
"Bold reading": "Odważne czytanie",
"SHOW EDITS": "POKAŻ EDYCJE"
}

3
translations/pt.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Webbots permitidos",
"Known Search Bots": "Bots de pesquisa na Web conhecidos",
"mitm": "A mensagem pode ter sido lida ou modificada por terceiros",
"Bold reading": "Leitura em negrito"
"Bold reading": "Leitura em negrito",
"SHOW EDITS": "MOSTRAR EDIÇÕES"
}

3
translations/ru.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Веб-боты разрешены",
"Known Search Bots": "Известные боты веб-поиска",
"mitm": "Сообщение могло быть прочитано или изменено третьим лицом",
"Bold reading": "Смелое чтение"
"Bold reading": "Смелое чтение",
"SHOW EDITS": "ПОКАЗАТЬ РЕДАКТИРОВАНИЕ"
}

3
translations/sw.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Mtandao wa Boti Unaruhusiwa",
"Known Search Bots": "Vijibu vya Utafutaji wa Wavuti vinavyojulikana",
"mitm": "Ujumbe ungeweza kusomwa au kurekebishwa na mtu mwingine",
"Bold reading": "Kusoma kwa ujasiri"
"Bold reading": "Kusoma kwa ujasiri",
"SHOW EDITS": "ONYESHA MABADILIKO"
}

3
translations/uk.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "Веб-боти дозволені",
"Known Search Bots": "Відомі пошукові роботи в Інтернеті",
"mitm": "Повідомлення могло бути прочитане або змінене третьою стороною",
"Bold reading": "Сміливе читання"
"Bold reading": "Сміливе читання",
"SHOW EDITS": "ПОКАЗАТИ ЗМІНИ"
}

3
translations/zh.json
View File

@ -519,5 +519,6 @@
"Web Bots Allowed": "允许网络机器人",
"Known Search Bots": "已知的网络搜索机器人",
"mitm": "消息可能已被第三方阅读或修改",
"Bold reading": "大胆阅读"
"Bold reading": "大胆阅读",
"SHOW EDITS": "显示编辑"
}

15
webapp_post.py
View File

@ -58,6 +58,7 @@ from utils import get_domain_from_actor
from utils import acct_dir
from utils import local_actor_url
from utils import is_unlisted_post
from content import create_edits_html
from content import bold_reading_string
from content import limit_repeated_words
from content import replace_emoji_from_tags
@ -1479,6 +1480,18 @@ def individual_post_as_html(signing_priv_key_pem: str,
_log_post_timing(enable_timing_log, post_start_time, '2')
# does this post have edits?
edits_post_url = \
remove_id_ending(message_id.strip()).replace('/', '#') + '.edits'
account_dir = acct_dir(base_dir, nickname, domain) + '/'
edits_filename = account_dir + box_name + '/' + edits_post_url
edits_str = ''
if os.path.isfile(edits_filename):
edits_json = load_json(edits_filename, 0, 1)
if edits_json:
edits_str = create_edits_html(edits_json, post_json_object,
translate, timezone)
message_id_str = ''
if message_id:
message_id_str = ';' + message_id
@ -2016,6 +2029,8 @@ def individual_post_as_html(signing_priv_key_pem: str,
if not is_pgp_encrypted(content_str):
if not is_patch:
# append any edits
content_str += edits_str
# Add bold text
if bold_reading and \
not displaying_ciphertext and \