From 800464d8956d03ef5c5c1a8eba9af86c00b3f201 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 17:28:14 +0000 Subject: [PATCH 1/9] Getting instance actor file --- utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.py b/utils.py index 8b7eca7ca..add6b09af 100644 --- a/utils.py +++ b/utils.py @@ -695,6 +695,8 @@ def save_json(json_object: {}, filename: str) -> bool: def load_json(filename: str, delay_sec: int = 2, max_tries: int = 5) -> {}: """Makes a few attempts to load a json formatted file """ + if '/Actor@' in filename: + filename = filename.replace('/Actor@', '/inbox@') json_object = None tries = 0 while tries < max_tries: From 8538b28b7debb43fe89ab32f0088a6e5db6e809c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 17:34:00 +0000 Subject: [PATCH 2/9] Getting instance actor file --- utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.py b/utils.py index add6b09af..5b7064ada 100644 --- a/utils.py +++ b/utils.py @@ -718,6 +718,8 @@ def load_json_onionify(filename: str, domain: str, onion_domain: str, """Makes a few attempts to load a json formatted file This also converts the domain name to the onion domain """ + if '/Actor@' in filename: + filename = filename.replace('/Actor@', '/inbox@') json_object = None tries = 0 while tries < 5: From 8978b2c2a2d61114d32c0b045de0e172c2898684 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 18:06:53 +0000 Subject: [PATCH 3/9] Mastodon api busy flag --- daemon.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/daemon.py b/daemon.py index 0c349b1ce..5313515e7 100644 --- a/daemon.py +++ b/daemon.py @@ -1060,16 +1060,53 @@ class PubServer(BaseHTTPRequestHandler): system_language: str, project_version: str, custom_emoji: [], - show_node_info_accounts: bool) -> bool: + show_node_info_accounts: bool, + referer_domain: str, + debug: bool) -> bool: """This is a vestigil mastodon API for the purpose of returning an empty result to sites like https://mastopeek.app-dist.eu """ if not path.startswith('/api/v1/'): return False + + if not referer_domain: + if not debug and not self.server.unit_test: + print('masto api request has no referer domain ' + str(ua_str)) + self._400() + return True + if referer_domain == self.server.domain_full: + print('masto api request from self') + self._400() + return True + if self.server.masto_api_is_active: + print('masto api is busy during request from ' + referer_domain) + self._503() + return True + self.server.masto_api_is_active = True + # is this a real website making the call ? + if not debug and not self.server.unit_test and referer_domain: + # Does calling_domain look like a domain? + if ' ' in referer_domain or \ + ';' in referer_domain or \ + '.' not in referer_domain: + print('masto api referer domain does not look like a domain ' + + referer_domain) + self._400() + self.server.masto_api_is_active = False + return True + if not self.server.allow_local_network_access: + if local_network_host(referer_domain): + print('masto api referer domain is from the ' + + 'local network ' + referer_domain) + self._400() + self.server.masto_api_is_active = False + return True + print('mastodon api v1: ' + path) print('mastodon api v1: authorized ' + str(authorized)) print('mastodon api v1: nickname ' + str(nickname)) + print('mastodon api v1: referer ' + referer_domain) self._update_known_crawlers(ua_str) broch_mode = broch_mode_is_active(base_dir) @@ -1108,10 +1145,12 @@ class PubServer(BaseHTTPRequestHandler): self._write(msg) if send_json_str: print(send_json_str) + self.server.masto_api_is_active = False return True # no api endpoints were matched self._404() + self.server.masto_api_is_active = False return True def _masto_api(self, path: str, calling_domain: str, @@ -1125,13 +1164,15 @@ class PubServer(BaseHTTPRequestHandler): system_language: str, project_version: str, custom_emoji: [], - show_node_info_accounts: bool) -> bool: + show_node_info_accounts: bool, + referer_domain: str, debug: bool) -> bool: return self._masto_api_v1(path, calling_domain, ua_str, authorized, http_prefix, base_dir, nickname, domain, domain_full, onion_domain, i2p_domain, translate, registration, system_language, project_version, custom_emoji, - show_node_info_accounts) + show_node_info_accounts, + referer_domain, debug) def _nodeinfo(self, ua_str: str, calling_domain: str, referer_domain: str, @@ -13910,7 +13951,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.system_language, self.server.project_version, self.server.custom_emoji, - self.server.show_node_info_accounts): + self.server.show_node_info_accounts, + referer_domain, + self.server.debug): return fitness_performance(getreq_start_time, self.server.fitness, @@ -18857,6 +18900,7 @@ def run_daemon(dyslexic_font: bool, httpd.post_to_nickname = None httpd.nodeinfo_is_active = False + httpd.masto_api_is_active = False httpd.dyslexic_font = dyslexic_font From 42d48470498a20947fd4ffaafaba59f55d25ce1e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 18:15:58 +0000 Subject: [PATCH 4/9] mastodon --- daemon.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/daemon.py b/daemon.py index 5313515e7..371b36696 100644 --- a/daemon.py +++ b/daemon.py @@ -1072,15 +1072,17 @@ class PubServer(BaseHTTPRequestHandler): if not referer_domain: if not debug and not self.server.unit_test: - print('masto api request has no referer domain ' + str(ua_str)) + print('mastodon api request has no referer domain ' + + str(ua_str)) self._400() return True if referer_domain == self.server.domain_full: - print('masto api request from self') + print('mastodon api request from self') self._400() return True if self.server.masto_api_is_active: - print('masto api is busy during request from ' + referer_domain) + print('mastodon api is busy during request from ' + + referer_domain) self._503() return True self.server.masto_api_is_active = True @@ -1090,14 +1092,15 @@ class PubServer(BaseHTTPRequestHandler): if ' ' in referer_domain or \ ';' in referer_domain or \ '.' not in referer_domain: - print('masto api referer domain does not look like a domain ' + + print('mastodon api ' + + 'referer domain does not look like a domain ' + referer_domain) self._400() self.server.masto_api_is_active = False return True if not self.server.allow_local_network_access: if local_network_host(referer_domain): - print('masto api referer domain is from the ' + + print('mastodon api referer domain is from the ' + 'local network ' + referer_domain) self._400() self.server.masto_api_is_active = False From 56acb826b78bc6746b09147cc25d74a2b63a9e1d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 18:19:26 +0000 Subject: [PATCH 5/9] Duplicate domain --- daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 371b36696..dd332a77c 100644 --- a/daemon.py +++ b/daemon.py @@ -1093,7 +1093,7 @@ class PubServer(BaseHTTPRequestHandler): ';' in referer_domain or \ '.' not in referer_domain: print('mastodon api ' + - 'referer domain does not look like a domain ' + + 'referer does not look like a domain ' + referer_domain) self._400() self.server.masto_api_is_active = False From e4f6a8e006d1710720bcf6a6453db87f0dc39f8e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 19:00:24 +0000 Subject: [PATCH 6/9] Check that mastodon api calls are from active sites --- daemon.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/daemon.py b/daemon.py index dd332a77c..6aaf849c5 100644 --- a/daemon.py +++ b/daemon.py @@ -1062,7 +1062,8 @@ class PubServer(BaseHTTPRequestHandler): custom_emoji: [], show_node_info_accounts: bool, referer_domain: str, - debug: bool) -> bool: + debug: bool, + calling_site_timeout: int) -> bool: """This is a vestigil mastodon API for the purpose of returning an empty result to sites like https://mastopeek.app-dist.eu @@ -1106,6 +1107,22 @@ class PubServer(BaseHTTPRequestHandler): self.server.masto_api_is_active = False return True + referer_url = http_prefix + '://' + referer_domain + if referer_domain + '/' in ua_str: + referer_url = referer_url + ua_str.split(referer_domain)[1] + if ' ' in referer_url: + referer_url = referer_url.split(' ')[0] + if ';' in referer_url: + referer_url = referer_url.split(';')[0] + if ')' in referer_url: + referer_url = referer_url.split(')')[0] + if not site_is_active(referer_url, calling_site_timeout): + print('nodeinfomastodon api referer url is not active ' + + referer_url) + self._400() + self.server.masto_api_is_active = False + return True + print('mastodon api v1: ' + path) print('mastodon api v1: authorized ' + str(authorized)) print('mastodon api v1: nickname ' + str(nickname)) @@ -1175,11 +1192,11 @@ class PubServer(BaseHTTPRequestHandler): translate, registration, system_language, project_version, custom_emoji, show_node_info_accounts, - referer_domain, debug) + referer_domain, debug, 5) def _nodeinfo(self, ua_str: str, calling_domain: str, referer_domain: str, - httpPrefix: str, calling_site_timeout: int, + http_prefix: str, calling_site_timeout: int, debug: bool) -> bool: if self.path.startswith('/nodeinfo/1.0'): self._400() @@ -1219,7 +1236,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.nodeinfo_is_active = False return True - referer_url = httpPrefix + '://' + referer_domain + referer_url = http_prefix + '://' + referer_domain if referer_domain + '/' in ua_str: referer_url = referer_url + ua_str.split(referer_domain)[1] if ' ' in referer_url: From 011f968a185542f06aea1cf5442c20b0129d4a99 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 4 Feb 2022 19:36:55 +0000 Subject: [PATCH 7/9] masto api --- daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 6aaf849c5..38e315ca0 100644 --- a/daemon.py +++ b/daemon.py @@ -1117,7 +1117,7 @@ class PubServer(BaseHTTPRequestHandler): if ')' in referer_url: referer_url = referer_url.split(')')[0] if not site_is_active(referer_url, calling_site_timeout): - print('nodeinfomastodon api referer url is not active ' + + print('mastodon api referer url is not active ' + referer_url) self._400() self.server.masto_api_is_active = False From 3a16200dbe037597c3602db8e1b72620647ea325 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 5 Feb 2022 10:49:31 +0000 Subject: [PATCH 8/9] Tidying --- daemon.py | 33 +++++++++------------------------ siteactive.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/daemon.py b/daemon.py index 38e315ca0..27179492a 100644 --- a/daemon.py +++ b/daemon.py @@ -366,7 +366,7 @@ from fitnessFunctions import fitness_performance from fitnessFunctions import fitness_thread from fitnessFunctions import sorted_watch_points from fitnessFunctions import html_watch_points_graph -from siteactive import site_is_active +from siteactive import referer_is_active import os @@ -1106,19 +1106,11 @@ class PubServer(BaseHTTPRequestHandler): self._400() self.server.masto_api_is_active = False return True - - referer_url = http_prefix + '://' + referer_domain - if referer_domain + '/' in ua_str: - referer_url = referer_url + ua_str.split(referer_domain)[1] - if ' ' in referer_url: - referer_url = referer_url.split(' ')[0] - if ';' in referer_url: - referer_url = referer_url.split(';')[0] - if ')' in referer_url: - referer_url = referer_url.split(')')[0] - if not site_is_active(referer_url, calling_site_timeout): + if not referer_is_active(http_prefix, + referer_domain, ua_str, + calling_site_timeout): print('mastodon api referer url is not active ' + - referer_url) + referer_domain) self._400() self.server.masto_api_is_active = False return True @@ -1236,18 +1228,11 @@ class PubServer(BaseHTTPRequestHandler): self.server.nodeinfo_is_active = False return True - referer_url = http_prefix + '://' + referer_domain - if referer_domain + '/' in ua_str: - referer_url = referer_url + ua_str.split(referer_domain)[1] - if ' ' in referer_url: - referer_url = referer_url.split(' ')[0] - if ';' in referer_url: - referer_url = referer_url.split(';')[0] - if ')' in referer_url: - referer_url = referer_url.split(')')[0] - if not site_is_active(referer_url, calling_site_timeout): + if not referer_is_active(http_prefix, + referer_domain, ua_str, + calling_site_timeout): print('nodeinfo referer url is not active ' + - referer_url) + referer_domain) self._400() self.server.nodeinfo_is_active = False return True diff --git a/siteactive.py b/siteactive.py index ebc4ab444..05e6e94f2 100644 --- a/siteactive.py +++ b/siteactive.py @@ -120,3 +120,18 @@ def site_is_active(url: str, timeout: int) -> bool: except BaseException: print('EX: site_is_active ' + str(loc)) return False + + +def referer_is_active(http_prefix: str, + referer_domain: str, ua_str: str, + calling_site_timeout: int) -> bool: + """Returns true if the given referer is an active website + """ + referer_url = http_prefix + '://' + referer_domain + if referer_domain + '/' in ua_str: + referer_url = referer_url + ua_str.split(referer_domain)[1] + ending_chars = (' ', ';', ')') + for end_ch in ending_chars: + if end_ch in referer_url: + referer_url = referer_url.split(end_ch)[0] + return site_is_active(referer_url, calling_site_timeout) From 4daec86f2a2e2053bfea561922c0d4542b478ee1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 6 Feb 2022 11:04:49 +0000 Subject: [PATCH 9/9] Support jpeg-xl format --- content.py | 1 + daemon.py | 5 +++++ metadata.py | 1 + newswire.py | 1 + outbox.py | 1 + person.py | 3 +++ session.py | 6 +++++- utils.py | 5 ++++- webapp_column_right.py | 3 ++- webapp_login.py | 3 +++ webapp_utils.py | 3 ++- 11 files changed, 28 insertions(+), 4 deletions(-) diff --git a/content.py b/content.py index 80fc3211a..aaa1545a3 100644 --- a/content.py +++ b/content.py @@ -1085,6 +1085,7 @@ def save_media_in_form_post(media_bytes, debug: bool, extension_list = { 'png': 'image/png', 'jpeg': 'image/jpeg', + 'jxl': 'image/jxl', 'gif': 'image/gif', 'svg': 'image/svg+xml', 'webp': 'image/webp', diff --git a/daemon.py b/daemon.py index 27179492a..c3a19c9ea 100644 --- a/daemon.py +++ b/daemon.py @@ -6861,6 +6861,9 @@ class PubServer(BaseHTTPRequestHandler): if 'image/avif' in self.headers['Accept']: fav_type = 'image/avif' fav_filename = fav_filename.split('.')[0] + '.avif' + if 'image/jxl' in self.headers['Accept']: + fav_type = 'image/jxl' + fav_filename = fav_filename.split('.')[0] + '.jxl' if not self.server.theme_name: self.theme_name = get_config_param(base_dir, 'theme') if not self.server.theme_name: @@ -6875,6 +6878,8 @@ class PubServer(BaseHTTPRequestHandler): fav_filename = fav_filename.replace('.webp', '.ico') elif fav_filename.endswith('.avif'): fav_filename = fav_filename.replace('.avif', '.ico') + elif fav_filename.endswith('.jxl'): + fav_filename = fav_filename.replace('.jxl', '.ico') if not os.path.isfile(favicon_filename): # default favicon favicon_filename = \ diff --git a/metadata.py b/metadata.py index a115732a7..6dbb88ff6 100644 --- a/metadata.py +++ b/metadata.py @@ -183,6 +183,7 @@ def meta_data_instance(showAccounts: bool, 'media_attachments': { 'supported_mime_types': [ 'image/jpeg', + 'image/jxl', 'image/png', 'image/gif', 'image/webp', diff --git a/newswire.py b/newswire.py index 3e04b4c45..c9734916e 100644 --- a/newswire.py +++ b/newswire.py @@ -164,6 +164,7 @@ def _download_newswire_feed_favicon(session, base_dir: str, 'ico': 'x-icon', 'png': 'png', 'jpg': 'jpeg', + 'jxl': 'jxl', 'gif': 'gif', 'avif': 'avif', 'svg': 'svg+xml', diff --git a/outbox.py b/outbox.py index 95932ebaf..db6e6a690 100644 --- a/outbox.py +++ b/outbox.py @@ -302,6 +302,7 @@ def post_message_to_outbox(session, translate: {}, extensions = { "jpeg": "jpg", + "jxl": "jxl", "gif": "gif", "svg": "svg", "webp": "webp", diff --git a/person.py b/person.py index bf3b54ea3..02f566159 100644 --- a/person.py +++ b/person.py @@ -140,6 +140,9 @@ def set_profile_image(base_dir: str, http_prefix: str, elif image_filename.endswith('.avif'): media_type = 'image/avif' icon_filename = icon_filename_base + '.avif' + elif image_filename.endswith('.jxl'): + media_type = 'image/jxl' + icon_filename = icon_filename_base + '.jxl' elif image_filename.endswith('.svg'): media_type = 'image/svg+xml' icon_filename = icon_filename_base + '.svg' diff --git a/session.py b/session.py index 2ced4f96f..e4c4aa69e 100644 --- a/session.py +++ b/session.py @@ -401,7 +401,7 @@ def post_image(session, attach_image_filename: str, federation_list: [], return None if not is_image_file(attach_image_filename): - print('Image must be png, jpg, webp, avif, gif or svg') + print('Image must be png, jpg, jxl, webp, avif, gif or svg') return None if not os.path.isfile(attach_image_filename): print('Image not found: ' + attach_image_filename) @@ -415,6 +415,8 @@ def post_image(session, attach_image_filename: str, federation_list: [], content_type = 'image/webp' elif attach_image_filename.endswith('.avif'): content_type = 'image/avif' + elif attach_image_filename.endswith('.jxl'): + content_type = 'image/jxl' elif attach_image_filename.endswith('.svg'): content_type = 'image/svg+xml' headers['Content-type'] = content_type @@ -469,6 +471,7 @@ def download_image(session, base_dir: str, url: str, 'png': 'png', 'jpg': 'jpeg', 'jpeg': 'jpeg', + 'jxl': 'jxl', 'gif': 'gif', 'svg': 'svg+xml', 'webp': 'webp', @@ -575,6 +578,7 @@ def download_image_any_mime_type(session, url: str, 'ico': 'x-icon', 'png': 'png', 'jpg': 'jpeg', + 'jxl': 'jxl', 'jpeg': 'jpeg', 'gif': 'gif', 'svg': 'svg+xml', diff --git a/utils.py b/utils.py index 5b7064ada..5def9ec8f 100644 --- a/utils.py +++ b/utils.py @@ -375,7 +375,7 @@ def get_audio_extensions() -> []: def get_image_extensions() -> []: """Returns a list of the possible image file extensions """ - return ('png', 'jpg', 'jpeg', 'gif', 'webp', 'avif', 'svg', 'ico') + return ('png', 'jpg', 'jpeg', 'gif', 'webp', 'avif', 'svg', 'ico', 'jxl') def get_image_mime_type(image_filename: str) -> str: @@ -384,6 +384,7 @@ def get_image_mime_type(image_filename: str) -> str: extensions_to_mime = { 'png': 'png', 'jpg': 'jpeg', + 'jxl': 'jxl', 'gif': 'gif', 'avif': 'avif', 'svg': 'svg+xml', @@ -402,6 +403,7 @@ def get_image_extension_from_mime_type(content_type: str) -> str: image_media = { 'png': 'png', 'jpeg': 'jpg', + 'jxl': 'jxl', 'gif': 'gif', 'svg+xml': 'svg', 'webp': 'webp', @@ -2535,6 +2537,7 @@ def media_file_mime_type(filename: str) -> str: 'json': 'application/json', 'png': 'image/png', 'jpg': 'image/jpeg', + 'jxl': 'image/jxl', 'jpeg': 'image/jpeg', 'gif': 'image/gif', 'svg': 'image/svg+xml', diff --git a/webapp_column_right.py b/webapp_column_right.py index 51527614c..b46473e25 100644 --- a/webapp_column_right.py +++ b/webapp_column_right.py @@ -249,7 +249,8 @@ def _html_newswire(base_dir: str, newswire: {}, nickname: str, moderator: bool, favicon_url = \ cached_favicon_filename.replace(base_dir, '') else: - extensions = ('png', 'jpg', 'gif', 'avif', 'svg', 'webp') + extensions = \ + ('png', 'jpg', 'gif', 'avif', 'svg', 'webp', 'jxl') for ext in extensions: cached_favicon_filename = \ get_fav_filename_from_url(base_dir, favicon_url) diff --git a/webapp_login.py b/webapp_login.py index ddf4308cd..988d47229 100644 --- a/webapp_login.py +++ b/webapp_login.py @@ -85,6 +85,9 @@ def html_login(css_cache: {}, translate: {}, elif os.path.isfile(base_dir + '/accounts/login.avif'): login_image = 'login.avif' login_image_filename = base_dir + '/accounts/' + login_image + elif os.path.isfile(base_dir + '/accounts/login.jxl'): + login_image = 'login.jxl' + login_image_filename = base_dir + '/accounts/' + login_image if not login_image_filename: login_image_filename = base_dir + '/accounts/' + login_image diff --git a/webapp_utils.py b/webapp_utils.py index 9b884f735..3ea3d5c06 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -251,6 +251,7 @@ def update_avatar_image_cache(signing_priv_key_pem: str, image_formats = { 'png': 'png', 'jpg': 'jpeg', + 'jxl': 'jxl', 'jpeg': 'jpeg', 'gif': 'gif', 'svg': 'svg+xml', @@ -998,7 +999,7 @@ def _is_attached_image(attachment_filename: str) -> bool: if '.' not in attachment_filename: return False image_ext = ( - 'png', 'jpg', 'jpeg', 'webp', 'avif', 'svg', 'gif' + 'png', 'jpg', 'jpeg', 'webp', 'avif', 'svg', 'gif', 'jxl' ) ext = attachment_filename.split('.')[-1] if ext in image_ext: