From bcedcd3f06ca0cc6fa1b16372a0bcc7d0777c0e0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Apr 2022 14:03:40 +0100 Subject: [PATCH 1/8] Don't speak emoji eithin display names --- blog.py | 2 +- content.py | 21 +++++++++++++++++---- daemon.py | 3 ++- posts.py | 2 +- tests.py | 20 +++++++++++++++++--- webapp_post.py | 2 +- webapp_utils.py | 4 ++-- 7 files changed, 41 insertions(+), 13 deletions(-) diff --git a/blog.py b/blog.py index 73601f043..078d50ff7 100644 --- a/blog.py +++ b/blog.py @@ -284,7 +284,7 @@ def _html_blog_post_content(debug: bool, session, authorized: bool, content_str = replace_emoji_from_tags(session, base_dir, content_str, post_json_object_tags, - 'content', debug) + 'content', debug, True) if article_added: blog_str += '
' + content_str + '\n' else: diff --git a/content.py b/content.py index 9b7e462fc..07f9885a4 100644 --- a/content.py +++ b/content.py @@ -398,7 +398,7 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None: def replace_emoji_from_tags(session, base_dir: str, content: str, tag: [], message_type: str, - debug: bool) -> str: + debug: bool, screen_readable: bool) -> str: """Uses the tags to replace :emoji: with html image markup """ for tag_item in tag: @@ -429,8 +429,13 @@ def replace_emoji_from_tags(session, base_dir: str, replaced = False try: replace_char = chr(int("0x" + icon_name, 16)) - content = content.replace(tag_item['name'], - replace_char) + if not screen_readable: + replace_char = \ + '' + content = \ + content.replace(tag_item['name'], + replace_char) replaced = True except BaseException: if debug: @@ -479,6 +484,10 @@ def replace_emoji_from_tags(session, base_dir: str, _update_common_emoji(base_dir, "0x" + icon_name) if icon_code_sequence: + if not screen_readable: + icon_code_sequence = \ + '' content = content.replace(tag_item['name'], icon_code_sequence) @@ -487,8 +496,12 @@ def replace_emoji_from_tags(session, base_dir: str, html_class = 'emojiheader' if message_type == 'profile': html_class = 'emojiprofile' + if screen_readable: + emoji_tag_name = tag_item['name'].replace(':', '') + else: + emoji_tag_name = '' emoji_html = "\""" content = content.replace(tag_item['name'], emoji_html) return content diff --git a/daemon.py b/daemon.py index c5e029fce..631a52083 100644 --- a/daemon.py +++ b/daemon.py @@ -18856,7 +18856,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.base_dir, fields['message'], tags, 'content', - self.server.debug) + self.server.debug, + True) post_json_object['object']['content'] = \ fields['message'] diff --git a/posts.py b/posts.py index 00befe37d..1f64d3f15 100644 --- a/posts.py +++ b/posts.py @@ -1445,7 +1445,7 @@ def _create_post_base(base_dir: str, if nickname != 'news': content = \ replace_emoji_from_tags(None, base_dir, content, tags, 'content', - False) + False, True) # remove replaced emoji hashtags_dict_copy = hashtags_dict.copy() for tag_name, tag in hashtags_dict_copy.items(): diff --git a/tests.py b/tests.py index 35d2cfaa6..ce15c5e7e 100644 --- a/tests.py +++ b/tests.py @@ -3673,9 +3673,23 @@ def _test_addemoji(base_dir: str): tags.append(tag) content = content_modified content_modified = \ - replace_emoji_from_tags(None, base_dir, content, tags, 'content', True) - # print('content_modified: ' + content_modified) - assert content_modified == '

Emoji 🍋 🍓 🍌

' + replace_emoji_from_tags(None, base_dir, content, tags, 'content', + True, True) + expected_content = '

Emoji 🍋 🍓 🍌

' + if content_modified != expected_content: + print('expected_content: ' + expected_content) + print('content_modified: ' + content_modified) + assert content_modified == expected_content + content_modified = \ + replace_emoji_from_tags(None, base_dir, content, tags, 'content', + True, False) + expected_content = '

Emoji ' + \ + ' ' + \ + '

' + if content_modified != expected_content: + print('expected_content: ' + expected_content) + print('content_modified: ' + content_modified) + assert content_modified == expected_content os.chdir(base_dir_original) shutil.rmtree(base_dir_original + '/.tests', diff --git a/webapp_post.py b/webapp_post.py index afafe1e4e..4716c3a11 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -2109,7 +2109,7 @@ def individual_post_as_html(signing_priv_key_pem: str, content_str = \ replace_emoji_from_tags(session, base_dir, content_str, post_json_object['object']['tag'], - 'content', False) + 'content', False, True) if is_muted: content_str = '' diff --git a/webapp_utils.py b/webapp_utils.py index b834cc6af..3c57e913b 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -984,12 +984,12 @@ def add_emoji_to_display_name(session, base_dir: str, http_prefix: str, display_name = \ replace_emoji_from_tags(session, base_dir, display_name, emoji_tags_list, - 'post header', False) + 'post header', False, False) else: display_name = \ replace_emoji_from_tags(session, base_dir, display_name, emoji_tags_list, 'profile', - False) + False, False) # print('TAG: display_name after tags 2: ' + display_name) # remove any stray emoji From 50e44ce98e7e9ce825ca5afc5cc111fce826aa7a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Apr 2022 14:19:12 +0100 Subject: [PATCH 2/8] Extra speaker substitutes --- speaker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/speaker.py b/speaker.py index c128ad019..a85af2198 100644 --- a/speaker.py +++ b/speaker.py @@ -106,11 +106,15 @@ def _speaker_pronounce(base_dir: str, say_text: str, translate: {}) -> str: "Memes": "Meemes", "rofl": "roll on the floor laughing", "ROFL": "roll on the floor laughing", + "lmao": "laughing", + "LMAO": "laughing", "fwiw": "for what it's worth", "fyi": "for your information", "irl": "in real life", "IRL": "in real life", "imho": "in my opinion", + "afaik": "as far as I know", + "AFAIK": "as far as I know", "fediverse": "fediiverse", "Fediverse": "Fediiverse", " foss ": " free and open source software ", From 84697cef2bd0b7d74f6584df3fcf3af7626b09cf Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Apr 2022 14:36:39 +0100 Subject: [PATCH 3/8] Double backslash --- speaker.py | 9 +++++---- translations/ar.json | 3 ++- translations/ca.json | 3 ++- translations/cy.json | 3 ++- translations/de.json | 3 ++- translations/en.json | 3 ++- translations/es.json | 3 ++- translations/fr.json | 3 ++- translations/ga.json | 3 ++- translations/hi.json | 3 ++- translations/it.json | 3 ++- translations/ja.json | 3 ++- translations/ko.json | 3 ++- translations/ku.json | 3 ++- translations/oc.json | 3 ++- translations/pl.json | 3 ++- translations/pt.json | 3 ++- translations/ru.json | 3 ++- translations/sw.json | 3 ++- translations/uk.json | 3 ++- translations/zh.json | 3 ++- 21 files changed, 45 insertions(+), 24 deletions(-) diff --git a/speaker.py b/speaker.py index a85af2198..b35a2cd0c 100644 --- a/speaker.py +++ b/speaker.py @@ -104,10 +104,10 @@ def _speaker_pronounce(base_dir: str, say_text: str, translate: {}) -> str: "Tor": "Toor", "memes": "meemes", "Memes": "Meemes", - "rofl": "roll on the floor laughing", - "ROFL": "roll on the floor laughing", - "lmao": "laughing", - "LMAO": "laughing", + "rofl": translate["laughing"], + "ROFL": translate["laughing"], + "lmao": translate["laughing"], + "LMAO": translate["laughing"], "fwiw": "for what it's worth", "fyi": "for your information", "irl": "in real life", @@ -128,6 +128,7 @@ def _speaker_pronounce(base_dir: str, say_text: str, translate: {}) -> str: "#nowplaying": translate["hashtag"] + " now-playing", "#NowPlaying": translate["hashtag"] + " now-playing", "#": translate["hashtag"] + ' ', + "¯\\_(ツ)_/¯": translate["shrug"], ":D": '. ' + translate["laughing"], ":-D": '. ' + translate["laughing"], ":)": '. ' + translate["smile"], diff --git a/translations/ar.json b/translations/ar.json index b1e77a64b..e1a2eb44d 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -546,5 +546,6 @@ "lang_sw": "السواحيلية", "lang_zh": "صينى", "Common emoji": "الرموز التعبيرية الشائعة", - "Copy and paste into your text": "نسخ ولصق في النص الخاص بك" + "Copy and paste into your text": "نسخ ولصق في النص الخاص بك", + "shrug": "هز كتفيه" } diff --git a/translations/ca.json b/translations/ca.json index 1c250ca7e..9d8833dc1 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -546,5 +546,6 @@ "lang_sw": "Suahili", "lang_zh": "Xinès", "Common emoji": "Emoji comú", - "Copy and paste into your text": "Copia i enganxa al teu text" + "Copy and paste into your text": "Copia i enganxa al teu text", + "shrug": "arronsar les espatlles" } diff --git a/translations/cy.json b/translations/cy.json index 4df858f12..b560e80ff 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "Tseiniaidd", "Common emoji": "Emoji cyffredin", - "Copy and paste into your text": "Copïwch a gludwch i'ch testun" + "Copy and paste into your text": "Copïwch a gludwch i'ch testun", + "shrug": "shrug" } diff --git a/translations/de.json b/translations/de.json index ebe210e6b..c47cdf0ef 100644 --- a/translations/de.json +++ b/translations/de.json @@ -546,5 +546,6 @@ "lang_sw": "Suaheli", "lang_zh": "Chinesisch", "Common emoji": "Gewöhnliches Emoji", - "Copy and paste into your text": "Kopieren und in Ihren Text einfügen" + "Copy and paste into your text": "Kopieren und in Ihren Text einfügen", + "shrug": "zucken" } diff --git a/translations/en.json b/translations/en.json index 059f1ea90..295551969 100644 --- a/translations/en.json +++ b/translations/en.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "Chinese", "Common emoji": "Common emoji", - "Copy and paste into your text": "Copy and paste into your text" + "Copy and paste into your text": "Copy and paste into your text", + "shrug": "shrug" } diff --git a/translations/es.json b/translations/es.json index 8f8ded93e..861013de4 100644 --- a/translations/es.json +++ b/translations/es.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "China", "Common emoji": "Emoticonos comunes", - "Copy and paste into your text": "Copia y pega en tu texto" + "Copy and paste into your text": "Copia y pega en tu texto", + "shrug": "encogimiento de hombros" } diff --git a/translations/fr.json b/translations/fr.json index 869b5a612..480e522c1 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "Chinoise", "Common emoji": "Émoji commun", - "Copy and paste into your text": "Copiez et collez dans votre texte" + "Copy and paste into your text": "Copiez et collez dans votre texte", + "shrug": "hausser les épaules" } diff --git a/translations/ga.json b/translations/ga.json index 1b9e61da3..b6fb83e86 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "Síneach", "Common emoji": "Emoji coitianta", - "Copy and paste into your text": "Cóipeáil agus greamaigh isteach i do théacs" + "Copy and paste into your text": "Cóipeáil agus greamaigh isteach i do théacs", + "shrug": "shrug" } diff --git a/translations/hi.json b/translations/hi.json index 41c166e2b..539f025f8 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "चीनी", "Common emoji": "आम इमोजी", - "Copy and paste into your text": "अपने टेक्स्ट में कॉपी और पेस्ट करें" + "Copy and paste into your text": "अपने टेक्स्ट में कॉपी और पेस्ट करें", + "shrug": "कंधे उचकाने की क्रिया" } diff --git a/translations/it.json b/translations/it.json index a329d132c..cea417eb7 100644 --- a/translations/it.json +++ b/translations/it.json @@ -546,5 +546,6 @@ "lang_sw": "Swahili", "lang_zh": "Cinese", "Common emoji": "Emoji comuni", - "Copy and paste into your text": "Copia e incolla nel tuo testo" + "Copy and paste into your text": "Copia e incolla nel tuo testo", + "shrug": "scrollare le spalle" } diff --git a/translations/ja.json b/translations/ja.json index c2a12661e..8fe3eea7c 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -546,5 +546,6 @@ "lang_sw": "スワヒリ語", "lang_zh": "中国語", "Common emoji": "一般的な絵文字", - "Copy and paste into your text": "コピーしてテキストに貼り付けます" + "Copy and paste into your text": "コピーしてテキストに貼り付けます", + "shrug": "肩をすくめる" } diff --git a/translations/ko.json b/translations/ko.json index c9e30c908..dd85895cb 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -546,5 +546,6 @@ "lang_sw": "스와힐리어", "lang_zh": "중국인", "Common emoji": "일반적인 이모티콘", - "Copy and paste into your text": "텍스트에 복사하여 붙여넣기" + "Copy and paste into your text": "텍스트에 복사하여 붙여넣기", + "shrug": "어깨를 으쓱하다" } diff --git a/translations/ku.json b/translations/ku.json index 920756b66..0c113099f 100644 --- a/translations/ku.json +++ b/translations/ku.json @@ -546,5 +546,6 @@ "lang_sw": "Swahîlîyî", "lang_zh": "Çînî", "Common emoji": "Emojiyên hevpar", - "Copy and paste into your text": "Di nivîsa xwe de kopî bikin û bixin" + "Copy and paste into your text": "Di nivîsa xwe de kopî bikin û bixin", + "shrug": "şuştin" } diff --git a/translations/oc.json b/translations/oc.json index 09b274b18..f11b3c7e3 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -542,5 +542,6 @@ "lang_sw": "Swahili", "lang_zh": "Chinese", "Common emoji": "Common emoji", - "Copy and paste into your text": "Copy and paste into your text" + "Copy and paste into your text": "Copy and paste into your text", + "shrug": "shrug" } diff --git a/translations/pl.json b/translations/pl.json index f88f3d844..75e7b0d51 100644 --- a/translations/pl.json +++ b/translations/pl.json @@ -546,5 +546,6 @@ "lang_sw": "Suahili", "lang_zh": "Chiński", "Common emoji": "Popularne emotikony", - "Copy and paste into your text": "Skopiuj i wklej do swojego tekstu" + "Copy and paste into your text": "Skopiuj i wklej do swojego tekstu", + "shrug": "wzruszać ramionami" } diff --git a/translations/pt.json b/translations/pt.json index 060083ce1..6654832c2 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -546,5 +546,6 @@ "lang_sw": "Suaíli", "lang_zh": "Chinês", "Common emoji": "Emoji comum", - "Copy and paste into your text": "Copie e cole no seu texto" + "Copy and paste into your text": "Copie e cole no seu texto", + "shrug": "dar de ombros" } diff --git a/translations/ru.json b/translations/ru.json index 2886a8258..1a5c51bce 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -546,5 +546,6 @@ "lang_sw": "суахили", "lang_zh": "Китайский", "Common emoji": "Общие смайлики", - "Copy and paste into your text": "Скопируйте и вставьте в свой текст" + "Copy and paste into your text": "Скопируйте и вставьте в свой текст", + "shrug": "пожимание плечами" } diff --git a/translations/sw.json b/translations/sw.json index e7412ba0c..0442893f4 100644 --- a/translations/sw.json +++ b/translations/sw.json @@ -546,5 +546,6 @@ "lang_sw": "Kiswahili", "lang_zh": "Kichina", "Common emoji": "Emoji ya kawaida", - "Copy and paste into your text": "Nakili na ubandike kwenye maandishi yako" + "Copy and paste into your text": "Nakili na ubandike kwenye maandishi yako", + "shrug": "piga mabega" } diff --git a/translations/uk.json b/translations/uk.json index a1afb5f8c..2e285f3c5 100644 --- a/translations/uk.json +++ b/translations/uk.json @@ -546,5 +546,6 @@ "lang_sw": "суахілі", "lang_zh": "китайський", "Common emoji": "Звичайні емодзі", - "Copy and paste into your text": "Скопіюйте та вставте у свій текст" + "Copy and paste into your text": "Скопіюйте та вставте у свій текст", + "shrug": "знизати плечима" } diff --git a/translations/zh.json b/translations/zh.json index 76cc3775c..7c9d98310 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -546,5 +546,6 @@ "lang_sw": "斯瓦希里语", "lang_zh": "中国人", "Common emoji": "常见表情符号", - "Copy and paste into your text": "复制并粘贴到您的文本中" + "Copy and paste into your text": "复制并粘贴到您的文本中", + "shrug": "耸耸肩" } From 637687ca2309e6d339b394ac5a1ab60fdfb0a78e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 22 Apr 2022 13:53:34 +0100 Subject: [PATCH 4/8] Parse alternate enclosures for podcast rss --- newswire.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- tests.py | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/newswire.py b/newswire.py index 089612524..9fcfe7ca9 100644 --- a/newswire.py +++ b/newswire.py @@ -623,11 +623,61 @@ def xml_podcast_to_dict(base_dir: str, xml_item: str, xml_str: str) -> {}: return podcast_properties -def get_link_from_rss_item(rss_item: str) -> (str, str): +def get_link_from_rss_item(rss_item: str, + preferred_mime_types: [] = None, + proxy_type: str = None) -> (str, str): """Extracts rss link from rss item string """ mime_type = None + if preferred_mime_types and ' None: '' + \ 'https://anchor.fm/creativecommons/episodes/' + \ 'Hessel-van-Oorschot-of-Tribe-of-Noise--Free-Music-Archive-e1crvce' + \ - '' + \ - 'Wed, 12 Jan 2022 14:28:46 GMT' + \ + '\n' + \ + 'Wed, 12 Jan 2022 14:28:46 GMT\n' + \ '' - link, mime_type = get_link_from_rss_item(rss_item) + 'ccb5edffbac1.mp3" length="67247880" type="audio/mpeg"/>\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + \ + '\n' + + link, mime_type = get_link_from_rss_item(rss_item, None, None) assert link - assert link.endswith('.mp3') + assert link.endswith('1.mp3') assert mime_type assert mime_type == 'audio/mpeg' + link, mime_type = get_link_from_rss_item(rss_item, ['audio/opus'], None) + assert mime_type + if mime_type != 'audio/opus': + print('mime_type: ' + mime_type) + assert mime_type == 'audio/opus' + assert link + assert link == 'https://whoframed.rodger/rabbit.opus' + + link, mime_type = get_link_from_rss_item(rss_item, ['audio/opus'], 'tor') + assert mime_type + if mime_type != 'audio/opus': + print('mime_type: ' + mime_type) + assert mime_type == 'audio/opus' + assert link + assert link == 'http://randomaddress.onion/rabbit.opus' + rss_item = \ '' + \ 'https://anchor.fm/creativecommons/episodes/' + \ From aa18c03acbb84f6dc6936d98d2ca392e72bfc612 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 22 Apr 2022 14:46:42 +0100 Subject: [PATCH 5/8] Command option for preferred podcast format --- daemon.py | 7 +++++- epicyon.py | 21 ++++++++++++++-- newsdaemon.py | 3 ++- newswire.py | 67 ++++++++++++++++++++++++++++++++++++++------------- tests.py | 24 ++++++++++++------ 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/daemon.py b/daemon.py index 631a52083..62a12a720 100644 --- a/daemon.py +++ b/daemon.py @@ -20628,7 +20628,8 @@ def load_tokens(base_dir: str, tokens_dict: {}, tokens_lookup: {}) -> None: break -def run_daemon(check_actor_timeout: int, +def run_daemon(preferred_podcast_formats: [], + check_actor_timeout: int, crawlers_allowed: [], dyslexic_font: bool, content_license_url: str, @@ -20720,6 +20721,10 @@ def run_daemon(check_actor_timeout: int, # scan the theme directory for any svg files containing scripts assert not scan_themes_for_scripts(base_dir) + # list of preferred podcast formats + # eg ['audio/opus', 'audio/mp3'] + httpd.preferred_podcast_formats = preferred_podcast_formats + # for each account, whether bold reading is enabled httpd.bold_reading = load_bold_reading(base_dir) diff --git a/epicyon.py b/epicyon.py index 0e5a1214c..16f584f7a 100644 --- a/epicyon.py +++ b/epicyon.py @@ -267,6 +267,10 @@ parser.add_argument('--proxy', dest='proxy_port', type=int, default=None, parser.add_argument('--path', dest='base_dir', type=str, default=os.getcwd(), help='Directory in which to store posts') +parser.add_argument('--podcast-formats', dest='podcast_formats', + type=str, default=None, + help='Preferred podcast formats separated by commas. ' + + 'eg. "opus, mp3"') parser.add_argument('--ytdomain', dest='yt_replace_domain', type=str, default=None, help='Domain used to replace youtube.com') @@ -1138,10 +1142,22 @@ if args.domain: domain = args.domain set_config_param(base_dir, 'domain', domain) +preferred_podcast_formats = None +if args.podcast_formats: + podcast_formats = args.podcast_formats.split(',') + for pod_format in podcast_formats: + pod_format = pod_format.lower().strip() + if '/' not in pod_format: + pod_format = 'audio/' + pod_format + if pod_format in preferred_podcast_formats: + continue + preferred_podcast_formats.append(pod_format) + if args.rss: session = create_session(None) testRSS = get_rss(base_dir, domain, session, args.rss, - False, False, 1000, 1000, 1000, 1000, debug) + False, False, 1000, 1000, 1000, 1000, debug, + preferred_podcast_formats) pprint(testRSS) sys.exit() @@ -3429,7 +3445,8 @@ if args.defaultCurrency: if __name__ == "__main__": print('allowdeletion: ' + str(args.allowdeletion)) - run_daemon(args.check_actor_timeout, + run_daemon(preferred_podcast_formats, + args.check_actor_timeout, crawlers_allowed, args.dyslexic_font, content_license_url, diff --git a/newsdaemon.py b/newsdaemon.py index 4d3209f6f..4ec7e0b1c 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -817,7 +817,8 @@ def run_newswire_daemon(base_dir: str, httpd, httpd.max_newswire_posts, httpd.maxCategoriesFeedItemSizeKb, httpd.system_language, - httpd.debug) + httpd.debug, + httpd.preferred_podcast_formats) if not httpd.newswire: print('Newswire feeds not updated') diff --git a/newswire.py b/newswire.py index 9fcfe7ca9..65c3d2302 100644 --- a/newswire.py +++ b/newswire.py @@ -624,8 +624,8 @@ def xml_podcast_to_dict(base_dir: str, xml_item: str, xml_str: str) -> {}: def get_link_from_rss_item(rss_item: str, - preferred_mime_types: [] = None, - proxy_type: str = None) -> (str, str): + preferred_mime_types: [], + proxy_type: str) -> (str, str): """Extracts rss link from rss item string """ mime_type = None @@ -717,7 +717,8 @@ def _xml2str_to_dict(base_dir: str, domain: str, xml_str: str, max_posts_per_source: int, max_feed_item_size_kb: int, max_categories_feedItem_size_kb: int, - session, debug: bool) -> {}: + session, debug: bool, + preferred_podcast_formats: []) -> {}: """Converts an xml RSS 2.0 string to a dictionary """ if '' not in xml_str: @@ -769,7 +770,15 @@ def _xml2str_to_dict(base_dir: str, domain: str, xml_str: str, description = description.split('')[0] description = remove_html(description) - link, link_mime_type = get_link_from_rss_item(rss_item) + proxy_type = None + if domain.endswith('.onion'): + proxy_type = 'tor' + elif domain.endswith('.i2p'): + proxy_type = 'i2p' + + link, link_mime_type = \ + get_link_from_rss_item(rss_item, preferred_podcast_formats, + proxy_type) if not link: continue @@ -812,7 +821,8 @@ def _xml1str_to_dict(base_dir: str, domain: str, xml_str: str, max_posts_per_source: int, max_feed_item_size_kb: int, max_categories_feedItem_size_kb: int, - session, debug: bool) -> {}: + session, debug: bool, + preferred_podcast_formats: []) -> {}: """Converts an xml RSS 1.0 string to a dictionary https://validator.w3.org/feed/docs/rss1.html """ @@ -866,7 +876,15 @@ def _xml1str_to_dict(base_dir: str, domain: str, xml_str: str, description = description.split('')[0] description = remove_html(description) - link, link_mime_type = get_link_from_rss_item(rss_item) + proxy_type = None + if domain.endswith('.onion'): + proxy_type = 'tor' + elif domain.endswith('.i2p'): + proxy_type = 'i2p' + + link, link_mime_type = \ + get_link_from_rss_item(rss_item, preferred_podcast_formats, + proxy_type) if not link: continue @@ -908,7 +926,8 @@ def _atom_feed_to_dict(base_dir: str, domain: str, xml_str: str, moderated: bool, mirrored: bool, max_posts_per_source: int, max_feed_item_size_kb: int, - session, debug: bool) -> {}: + session, debug: bool, + preferred_podcast_formats: []) -> {}: """Converts an atom feed string to a dictionary """ if '' not in xml_str: @@ -951,7 +970,15 @@ def _atom_feed_to_dict(base_dir: str, domain: str, xml_str: str, description = description.split('')[0] description = remove_html(description) - link, link_mime_type = get_link_from_rss_item(atom_item) + proxy_type = None + if domain.endswith('.onion'): + proxy_type = 'tor' + elif domain.endswith('.i2p'): + proxy_type = 'i2p' + + link, link_mime_type = \ + get_link_from_rss_item(atom_item, preferred_podcast_formats, + proxy_type) if not link: continue @@ -1155,7 +1182,7 @@ def _atom_feed_yt_to_dict(base_dir: str, domain: str, xml_str: str, description = description.split('')[0] description = remove_html(description) - link, _ = get_link_from_rss_item(atom_item) + link, _ = get_link_from_rss_item(atom_item, None, None) if not link: link = atom_item.split('')[1] link = link.split('')[0] @@ -1196,7 +1223,8 @@ def _xml_str_to_dict(base_dir: str, domain: str, xml_str: str, max_posts_per_source: int, max_feed_item_size_kb: int, max_categories_feedItem_size_kb: int, - session, debug: bool) -> {}: + session, debug: bool, + preferred_podcast_formats: []) -> {}: """Converts an xml string to a dictionary """ if '' in xml_str and '' in xml_str: @@ -1211,18 +1239,19 @@ def _xml_str_to_dict(base_dir: str, domain: str, xml_str: str, xml_str, moderated, mirrored, max_posts_per_source, max_feed_item_size_kb, max_categories_feedItem_size_kb, - session, debug) + session, debug, + preferred_podcast_formats) if ' {}: + max_categories_feedItem_size_kb: int, debug: bool, + preferred_podcast_formats: []) -> {}: """Returns an RSS url as a dict """ if not isinstance(url, str): @@ -1281,7 +1311,8 @@ def get_rss(base_dir: str, domain: str, session, url: str, max_posts_per_source, max_feed_item_size_kb, max_categories_feedItem_size_kb, - session, debug) + session, debug, + preferred_podcast_formats) else: print('WARN: feed is too large, ' + 'or contains invalid characters: ' + url) @@ -1526,7 +1557,8 @@ def get_dict_from_newswire(session, base_dir: str, domain: str, max_tags: int, max_feed_item_size_kb: int, max_newswire_posts: int, max_categories_feedItem_size_kb: int, - system_language: str, debug: bool) -> {}: + system_language: str, debug: bool, + preferred_podcast_formats: []) -> {}: """Gets rss feeds as a dictionary from newswire file """ subscriptions_filename = base_dir + '/accounts/newswire.txt' @@ -1567,7 +1599,8 @@ def get_dict_from_newswire(session, base_dir: str, domain: str, moderated, mirrored, max_posts_per_source, max_feed_size_kb, max_feed_item_size_kb, - max_categories_feedItem_size_kb, debug) + max_categories_feedItem_size_kb, debug, + preferred_podcast_formats) if items_list: for date_str, item in items_list.items(): result[date_str] = item diff --git a/tests.py b/tests.py index b5eed571e..1faa65882 100644 --- a/tests.py +++ b/tests.py @@ -827,8 +827,10 @@ def create_server_alice(path: str, domain: str, port: int, dyslexic_font = False crawlers_allowed = [] check_actor_timeout = 2 + preferred_podcast_formats = None print('Server running: Alice') - run_daemon(check_actor_timeout, + run_daemon(preferred_podcast_formats, + check_actor_timeout, crawlers_allowed, dyslexic_font, content_license_url, @@ -984,8 +986,10 @@ def create_server_bob(path: str, domain: str, port: int, dyslexic_font = False crawlers_allowed = [] check_actor_timeout = 2 + preferred_podcast_formats = None print('Server running: Bob') - run_daemon(check_actor_timeout, + run_daemon(preferred_podcast_formats, + check_actor_timeout, crawlers_allowed, dyslexic_font, content_license_url, @@ -1064,8 +1068,10 @@ def create_server_eve(path: str, domain: str, port: int, federation_list: [], dyslexic_font = False crawlers_allowed = [] check_actor_timeout = 2 + preferred_podcast_formats = None print('Server running: Eve') - run_daemon(check_actor_timeout, + run_daemon(preferred_podcast_formats, + check_actor_timeout, crawlers_allowed, dyslexic_font, content_license_url, @@ -1146,8 +1152,10 @@ def create_server_group(path: str, domain: str, port: int, dyslexic_font = False crawlers_allowed = [] check_actor_timeout = 2 + preferred_podcast_formats = None print('Server running: Group') - run_daemon(check_actor_timeout, + run_daemon(preferred_podcast_formats, + check_actor_timeout, crawlers_allowed, dyslexic_font, content_license_url, @@ -6789,7 +6797,7 @@ def _test_xml_podcast_dict(base_dir: str) -> None: assert len(podcast_properties['locations']) == 1 -def _test_get_link_from_rss_item() -> None: +def _test_link_from_rss_item() -> None: print('test_get_link_from_rssitem') rss_item = \ '' + \ @@ -6844,7 +6852,7 @@ def _test_get_link_from_rss_item() -> None: 'Hessel-van-Oorschot-of-Tribe-of-Noise--Free-Music-Archive-e1crvce' + \ '' + \ 'Wed, 12 Jan 2022 14:28:46 GMT' - link, mime_type = get_link_from_rss_item(rss_item) + link, mime_type = get_link_from_rss_item(rss_item, None, None) assert link assert link.startswith('https://anchor.fm') assert not mime_type @@ -6855,7 +6863,7 @@ def _test_get_link_from_rss_item() -> None: 'Hessel-van-Oorschot-of-Tribe-of-Noise--Free-Music-Archive-e1crvce' + \ '"/>' + \ 'Wed, 12 Jan 2022 14:28:46 GMT' - link, mime_type = get_link_from_rss_item(rss_item) + link, mime_type = get_link_from_rss_item(rss_item, None, None) assert link assert link.startswith('https://test.link/creativecommons') @@ -7071,7 +7079,7 @@ def run_all_tests(): _test_bold_reading() _test_published_to_local_timezone() _test_safe_webtext() - _test_get_link_from_rss_item() + _test_link_from_rss_item() _test_xml_podcast_dict(base_dir) _test_get_actor_from_in_reply_to() _test_valid_emoji_content() From 0c0f21d814f9d1cf971ab05f5acd352a145d4500 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 22 Apr 2022 15:09:41 +0100 Subject: [PATCH 6/8] Preferred podcast formats can be loaded from config --- epicyon.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/epicyon.py b/epicyon.py index 16f584f7a..81bc08951 100644 --- a/epicyon.py +++ b/epicyon.py @@ -1142,9 +1142,16 @@ if args.domain: domain = args.domain set_config_param(base_dir, 'domain', domain) +# comma separated list of preferred audio formats. eg. "opus", "mp3" preferred_podcast_formats = None if args.podcast_formats: - podcast_formats = args.podcast_formats.split(',') + podcast_formats_str = args.podcast_formats +else: + podcast_formats_str = \ + get_config_param(base_dir, 'preferred_podcast_formats') +if podcast_formats_str: + podcast_formats = podcast_formats_str.split(',') + preferred_podcast_formats = [] for pod_format in podcast_formats: pod_format = pod_format.lower().strip() if '/' not in pod_format: From 5446bcc54afe4f3857be1d2fb5f32d4cc9a1d09e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 22 Apr 2022 15:18:07 +0100 Subject: [PATCH 7/8] Camel case in config file --- epicyon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/epicyon.py b/epicyon.py index 81bc08951..728967887 100644 --- a/epicyon.py +++ b/epicyon.py @@ -1148,7 +1148,7 @@ if args.podcast_formats: podcast_formats_str = args.podcast_formats else: podcast_formats_str = \ - get_config_param(base_dir, 'preferred_podcast_formats') + get_config_param(base_dir, 'preferredPodcastFormats') if podcast_formats_str: podcast_formats = podcast_formats_str.split(',') preferred_podcast_formats = [] From 1e2b0e28df0d117292d5a2dbf6e62ebd85ee3414 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 22 Apr 2022 15:27:25 +0100 Subject: [PATCH 8/8] Default podcast audio formats --- epicyon.py | 3 ++- tests.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/epicyon.py b/epicyon.py index 728967887..034ae5b77 100644 --- a/epicyon.py +++ b/epicyon.py @@ -1143,7 +1143,8 @@ if args.domain: set_config_param(base_dir, 'domain', domain) # comma separated list of preferred audio formats. eg. "opus", "mp3" -preferred_podcast_formats = None +# in order of preference +preferred_podcast_formats = ['ogg', 'mpeg', 'opus'] if args.podcast_formats: podcast_formats_str = args.podcast_formats else: diff --git a/tests.py b/tests.py index 1faa65882..f0b1f7bab 100644 --- a/tests.py +++ b/tests.py @@ -6830,6 +6830,18 @@ def _test_link_from_rss_item() -> None: assert mime_type assert mime_type == 'audio/mpeg' + link, mime_type = get_link_from_rss_item(rss_item, ['audio/mp3'], None) + assert link + assert link.endswith('1.mp3') + assert mime_type + assert mime_type == 'audio/mpeg' + + link, mime_type = get_link_from_rss_item(rss_item, ['audio/mpeg'], None) + assert link + assert link == 'https://whoframed.rodger/rabbit.mp3' + assert mime_type + assert mime_type == 'audio/mpeg' + link, mime_type = get_link_from_rss_item(rss_item, ['audio/opus'], None) assert mime_type if mime_type != 'audio/opus':