From b8d294a5daaf30130a2a15016112bab0b22b3dc0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 15 Oct 2024 23:46:47 +0100 Subject: [PATCH] Resolve openstreetmap go paths --- cwlists/ai_generated.json | 2 +- daemon_get.py | 13 ++++-- daemon_post_question.py | 2 +- daemon_post_receive.py | 23 ++++++---- epicyon.py | 17 ++++---- inbox.py | 4 +- inbox_receive.py | 2 +- maps.py | 89 +++++++++++++++++++++++++++++++-------- newsdaemon.py | 12 +++--- outbox.py | 2 +- posts.py | 54 +++++++++++++----------- session.py | 25 +++++++++++ tests.py | 42 ++++++++++-------- webapp_calendar.py | 13 ++++-- webapp_hashtagswarm.py | 7 +-- webapp_post.py | 6 ++- webapp_search.py | 2 +- 17 files changed, 215 insertions(+), 100 deletions(-) diff --git a/cwlists/ai_generated.json b/cwlists/ai_generated.json index 35657dc95..b10fa40d0 100644 --- a/cwlists/ai_generated.json +++ b/cwlists/ai_generated.json @@ -545,7 +545,7 @@ "stablediffusionweb.com", "allabtai.com", "cozybrushery.com", - "story.com", + "/story.com", "copilot.microsoft.com", "copilot.github.com", "ubrand.com", diff --git a/daemon_get.py b/daemon_get.py index 9ce087d81..26749b196 100644 --- a/daemon_get.py +++ b/daemon_get.py @@ -3114,7 +3114,8 @@ def daemon_http_get(self) -> None: map_str = \ map_format_from_tagmaps_path(self.server.base_dir, self.path, self.server.map_format, - self.server.domain) + self.server.domain, + self.server.session) if map_str: msg = map_str.encode('utf-8') msglen = len(msg) @@ -3249,7 +3250,10 @@ def daemon_http_get(self) -> None: access_keys, False, self.server.system_language, self.server.default_timeline, - self.server.theme_name) + self.server.theme_name, + self.server.session, + self.server.session_onion, + self.server.session_i2p) if msg: msg = msg.encode('utf-8') msglen = len(msg) @@ -3292,7 +3296,10 @@ def daemon_http_get(self) -> None: True, self.server.system_language, self.server.default_timeline, - self.server.theme_name) + self.server.theme_name, + self.server.session, + self.server.session_onion, + self.server.session_i2p) if msg: msg = msg.encode('utf-8') msglen = len(msg) diff --git a/daemon_post_question.py b/daemon_post_question.py index 2537254fc..3971d0475 100644 --- a/daemon_post_question.py +++ b/daemon_post_question.py @@ -259,7 +259,7 @@ def _send_reply_to_question(self, base_dir: str, languages_understood, False, translate, buy_url, chat_url, - auto_cw_cache) + auto_cw_cache, curr_session) if message_json: # NOTE: content and contentMap are not required, but we will keep # them in there so that the post does not get filtered out by diff --git a/daemon_post_receive.py b/daemon_post_receive.py index 170618eab..3a7a993f2 100644 --- a/daemon_post_receive.py +++ b/daemon_post_receive.py @@ -181,7 +181,8 @@ def _receive_new_post_process_newpost(self, fields: {}, translate, buy_url, chat_url, auto_cw_cache, - fields['searchableByDropdown']) + fields['searchableByDropdown'], + curr_session) if message_json: if edited_postid: update_edited_post(base_dir, nickname, domain, @@ -339,7 +340,8 @@ def _receive_new_post_process_newblog(self, fields: {}, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - fields['searchableByDropdown']) + fields['searchableByDropdown'], + curr_session) if message_json: if fields['schedulePost']: return NEW_POST_SUCCESS @@ -579,7 +581,7 @@ def _receive_new_post_process_newunlisted(self, fields: {}, languages_understood, translate, buy_url, chat_url, - auto_cw_cache) + auto_cw_cache, curr_session) if message_json: if edited_postid: update_edited_post(base_dir, nickname, domain, @@ -741,7 +743,8 @@ def _receive_new_post_process_newfollowers(self, fields: {}, translate, buy_url, chat_url, auto_cw_cache, - fields['searchableByDropdown']) + fields['searchableByDropdown'], + curr_session) if message_json: if edited_postid: update_edited_post(base_dir, @@ -912,7 +915,7 @@ def _receive_new_post_process_newdm(self, fields: {}, reply_is_chat, translate, buy_url, chat_url, - auto_cw_cache) + auto_cw_cache, curr_session) if message_json: print('DEBUG: posting DM edited_postid ' + str(edited_postid)) @@ -1065,7 +1068,7 @@ def _receive_new_post_process_newreminder(self, fields: {}, nickname: str, languages_understood, False, translate, buy_url, chat_url, - auto_cw_cache) + auto_cw_cache, curr_session) if message_json: if fields['schedulePost']: return NEW_POST_SUCCESS @@ -1173,7 +1176,8 @@ def _receive_new_post_process_newreport(self, fields: {}, content_license_url, media_license_url, media_creator, languages_understood, - translate, auto_cw_cache) + translate, auto_cw_cache, + curr_session) if message_json: if post_to_outbox(self, message_json, project_version, @@ -1254,7 +1258,7 @@ def _receive_new_post_process_newquestion(self, fields: {}, media_license_url, media_creator, languages_understood, translate, - auto_cw_cache) + auto_cw_cache, curr_session) if message_json: if debug: print('DEBUG: new Question') @@ -1386,7 +1390,8 @@ def _receive_new_post_process_newreading(self, fields: {}, translate, buy_url, chat_url, auto_cw_cache, - fields['searchableByDropdown']) + fields['searchableByDropdown'], + curr_session) if message_json: if edited_postid: update_edited_post(base_dir, nickname, domain, diff --git a/epicyon.py b/epicyon.py index 61ca15e01..b79a0f0de 100644 --- a/epicyon.py +++ b/epicyon.py @@ -3651,6 +3651,7 @@ def _command_options() -> None: auto_cw_cache = {} test_video_transcript = '' searchable_by = [] + curr_session = None create_public_post(base_dir, nickname, domain, port, http_prefix, "like this is totally just a #test man", @@ -3669,7 +3670,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "Zoiks!!!", test_save_to_file, @@ -3687,7 +3688,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "Hey scoob we need like a hundred more #milkshakes", test_save_to_file, @@ -3705,7 +3706,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "Getting kinda spooky around here", test_save_to_file, @@ -3723,7 +3724,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "And they would have gotten away with it too" + "if it wasn't for those pesky hackers", @@ -3742,7 +3743,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "man these centralized sites are like the worst!", test_save_to_file, @@ -3760,7 +3761,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "another mystery solved #test", test_save_to_file, @@ -3778,7 +3779,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) create_public_post(base_dir, nickname, domain, port, http_prefix, "let's go bowling", test_save_to_file, @@ -3796,7 +3797,7 @@ def _command_options() -> None: low_bandwidth, argb.content_license_url, argb.media_license_url, argb.media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, curr_session) domain_full = domain + ':' + str(port) clear_follows(base_dir, nickname, domain, 'following.txt') follow_person(base_dir, nickname, domain, 'maxboardroom', domain_full, diff --git a/inbox.py b/inbox.py index 3eabc4e5f..91f11c323 100644 --- a/inbox.py +++ b/inbox.py @@ -1367,7 +1367,7 @@ def _bounce_dm(sender_post_id: str, session, http_prefix: str, dm_license_url, '', languages_understood, bounce_is_chat, translate, buy_url, chat_url, - auto_cw_cache) + auto_cw_cache, session) if not post_json_object: print('WARN: unable to create bounce message to ' + sending_handle) return False @@ -2525,7 +2525,7 @@ def _inbox_after_initial(server, inbox_start_time, store_hash_tags(base_dir, handle_name, domain, http_prefix, domain_full, - post_json_object, translate) + post_json_object, translate, session) fitness_performance(inbox_start_time, server.fitness, 'INBOX', 'store_hash_tags', diff --git a/inbox_receive.py b/inbox_receive.py index c05db35d6..8cb072906 100644 --- a/inbox_receive.py +++ b/inbox_receive.py @@ -1873,7 +1873,7 @@ def receive_announce(recent_posts_cache: {}, store_hash_tags(base_dir, nickname, domain, http_prefix, domain_full, - post_json_object, translate) + post_json_object, translate, session) # Try to obtain the actor for this person # so that their avatar can be shown lookup_actor = None diff --git a/maps.py b/maps.py index da3ec64a8..883497cc8 100644 --- a/maps.py +++ b/maps.py @@ -20,6 +20,7 @@ from utils import has_object_dict from utils import date_utcnow from utils import date_epoch from utils import date_from_string_format +from session import get_resolved_url def _geocoords_to_osm_link(osm_domain: str, zoom: int, @@ -136,7 +137,7 @@ def _geocoords_from_osmorg_link(url: str) -> (int, float, float): return None, None, None if 'mlon=' not in url: return None, None, None - if 'zoom=' not in url: + if 'zoom=' not in url and '#map=' not in url: return None, None, None latitude = url.split('mlat=')[1] @@ -151,9 +152,15 @@ def _geocoords_from_osmorg_link(url: str) -> (int, float, float): if not is_float(longitude): return None, None, None - zoom = url.split('zoom=')[1] - if '&' in zoom: - zoom = zoom.split('&')[0] + if 'zoom=' in url: + zoom = url.split('zoom=')[1] + if '&' in zoom: + zoom = zoom.split('&')[0] + else: + zoom = url.split('#map=')[1] + if '/' in zoom: + zoom = zoom.split('/')[0] + if not zoom.isdigit(): return None, None, None zoom = int(zoom) @@ -162,6 +169,35 @@ def _geocoords_from_osmorg_link(url: str) -> (int, float, float): return zoom, latitude, longitude +def _geocoords_from_osmorg_go_link(url: str, session) -> (int, float, float): + """Returns geocoordinates from an OSM go map link + """ + osm_domain = 'osm.org' + if osm_domain not in url: + return None, None, None + if 'mlat=' in url: + return None, None, None + if 'mlon=' in url: + return None, None, None + if '/go/' not in url: + return None, None, None + + # TODO resolve url equivalent to + # curl -Ls -o /dev/null -w %{url_effective} [url] + resolved_url = get_resolved_url(session, url) + + if not resolved_url: + return None, None, None + + if 'osm.org' in resolved_url: + (zoom, latitude, longitude) = \ + _geocoords_from_osmorg_link(resolved_url) + else: + (zoom, latitude, longitude) = \ + _geocoords_from_osm_link(resolved_url, 'openstreetmap.org') + return zoom, latitude, longitude + + def _geocoords_from_osmand_link(url: str) -> (int, float, float): """Returns geocoordinates from an OSM android map link """ @@ -383,14 +419,18 @@ def _geocoords_from_wego_link(url: str) -> (int, float, float): return zoom, latitude, longitude -def geocoords_from_map_link(url: str, - osm_domain: str) -> (int, float, float): +def geocoords_from_map_link(url: str, osm_domain: str, + session) -> (int, float, float): """Returns geocoordinates from a map link url """ if osm_domain in url: zoom, latitude, longitude = \ _geocoords_from_osm_link(url, osm_domain) return zoom, latitude, longitude + if 'osm.org' in url and 'mlat=' not in url and '/go/' in url: + zoom, latitude, longitude = \ + _geocoords_from_osmorg_go_link(url, session) + return zoom, latitude, longitude if 'osm.org' in url and 'mlat=' in url: zoom, latitude, longitude = \ _geocoords_from_osmorg_link(url) @@ -424,13 +464,20 @@ def geocoords_from_map_link(url: str, def html_open_street_map(url: str, bounding_box_degrees: float, - translate: {}, + translate: {}, session, + session_onion, session_i2p, width: str = "725", height: str = "650") -> str: """Returns embed html for an OSM link """ osm_domain = 'openstreetmap.org' - zoom, latitude, longitude = geocoords_from_map_link(url, osm_domain) + map_session = session + if '.onion/' in url: + map_session = session_onion + elif '.i2p/' in url: + map_session = session_i2p + zoom, latitude, longitude = \ + geocoords_from_map_link(url, osm_domain, map_session) if not latitude: return '' if not longitude: @@ -524,7 +571,7 @@ def get_map_preferences_coords(base_dir: str, nickname: str, return None, None, None -def get_map_links_from_post_content(content: str) -> []: +def get_map_links_from_post_content(content: str, session) -> []: """Returns a list of map links """ osm_domain = 'openstreetmap.org' @@ -542,7 +589,8 @@ def get_map_links_from_post_content(content: str) -> []: url = url.split('<')[0] if not url: continue - zoom, latitude, longitude = geocoords_from_map_link(url, osm_domain) + zoom, latitude, longitude = \ + geocoords_from_map_link(url, osm_domain, session) if not latitude: continue if not longitude: @@ -667,7 +715,7 @@ def _hashtag_map_to_format(base_dir: str, tag_name: str, start_hours_since_epoch: int, end_hours_since_epoch: int, nickname: str, domain: str, - map_format: str) -> str: + map_format: str, session) -> str: """Returns the KML/GPX for a given hashtag between the given times """ place_ctr = 0 @@ -704,7 +752,7 @@ def _hashtag_map_to_format(base_dir: str, tag_name: str, # get the geocoordinates from the map link map_link = link_line[1] zoom, latitude, longitude = \ - geocoords_from_map_link(map_link, osm_domain) + geocoords_from_map_link(map_link, osm_domain, session) if not zoom: continue if not latitude: @@ -739,7 +787,8 @@ def _hashtag_map_to_format(base_dir: str, tag_name: str, def _hashtag_map_within_hours(base_dir: str, tag_name: str, hours: int, map_format: str, - nickname: str, domain: str) -> str: + nickname: str, domain: str, + session) -> str: """Returns gpx/kml for a hashtag containing maps for the last number of hours """ @@ -753,7 +802,8 @@ def _hashtag_map_within_hours(base_dir: str, tag_name: str, _hashtag_map_to_format(base_dir, tag_name, start_hours_since_epoch, end_hours_since_epoch, - nickname, domain, map_format) + nickname, domain, map_format, + session) return map_str @@ -777,7 +827,7 @@ def _get_tagmaps_time_periods() -> {}: def map_format_from_tagmaps_path(base_dir: str, path: str, map_format: str, - domain: str) -> str: + domain: str, session) -> str: """Returns gpx/kml for a given tagmaps path /tagmaps/tagname-time_period """ @@ -802,13 +852,15 @@ def map_format_from_tagmaps_path(base_dir: str, path: str, nickname = nickname.split('/')[0] return _hashtag_map_within_hours(base_dir, tag_name, hours, map_format, - nickname, domain) + nickname, domain, + session) return None def html_hashtag_maps(base_dir: str, tag_name: str, translate: {}, map_format: str, - nickname: str, domain: str) -> str: + nickname: str, domain: str, + session) -> str: """Returns html for maps associated with a hashtag """ tag_map_filename = base_dir + '/tagmaps/' + tag_name + '.txt' @@ -822,7 +874,8 @@ def html_hashtag_maps(base_dir: str, tag_name: str, for period_str, hours in time_period.items(): new_map_str = \ _hashtag_map_within_hours(base_dir, tag_name, hours, - map_format, nickname, domain) + map_format, nickname, domain, + session) if not new_map_str: continue if new_map_str == map_str: diff --git a/newsdaemon.py b/newsdaemon.py index 182fbab47..e042340c2 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -345,7 +345,7 @@ def _hashtag_add(base_dir: str, http_prefix: str, domain_full: str, domain = domain.split(':')[0] store_hash_tags(base_dir, 'news', domain, http_prefix, domain_full, - post_json_object, translate) + post_json_object, translate, session) def _hashtag_remove(http_prefix: str, domain_full: str, post_json_object: {}, @@ -571,7 +571,8 @@ def _convert_rss_to_activitypub(base_dir: str, http_prefix: str, low_bandwidth: bool, content_license_url: str, media_license_url: str, - media_creator: str) -> None: + media_creator: str, + session) -> None: """Converts rss items in a newswire into posts """ if not newswire: @@ -675,7 +676,7 @@ def _convert_rss_to_activitypub(base_dir: str, http_prefix: str, content_license_url, media_license_url, media_creator, languages_understood, translate, - buy_url, chat_url) + buy_url, chat_url, session) if not blog: continue @@ -760,7 +761,7 @@ def _convert_rss_to_activitypub(base_dir: str, http_prefix: str, store_hash_tags(base_dir, 'news', domain, http_prefix, domain_full, - blog, translate) + blog, translate, session) clear_from_post_caches(base_dir, recent_posts_cache, post_id) if save_json(blog, filename): @@ -865,7 +866,8 @@ def run_newswire_daemon(base_dir: str, httpd, httpd.system_language, httpd.low_bandwidth, httpd.content_license_url, - httpd.content_license_url, '') + httpd.content_license_url, '', + httpd.session) print('Newswire feed converted to ActivityPub') if httpd.max_news_posts > 0: diff --git a/outbox.py b/outbox.py index d3dd12d6c..2c764edb7 100644 --- a/outbox.py +++ b/outbox.py @@ -523,7 +523,7 @@ def post_message_to_outbox(session, translate: {}, store_hash_tags(base_dir, post_to_nickname, domain, http_prefix, domain_full, - message_json, translate) + message_json, translate, session) # if this is a blog post or an event then save to its own box if message_json['type'] == 'Create': diff --git a/posts.py b/posts.py index 155e8a0c7..455698137 100644 --- a/posts.py +++ b/posts.py @@ -1521,7 +1521,7 @@ def _create_post_place_and_time(event_date: str, end_date: str, schedule_post: bool, event_uuid: str, location: str, - tags: []) -> str: + tags: [], session) -> str: """Adds a place and time to the tags on a new post """ end_date_str = None @@ -1579,7 +1579,8 @@ def _create_post_place_and_time(event_date: str, end_date: str, latitude = longitude = None if '://' in location: _, latitude, longitude = \ - geocoords_from_map_link(location, 'openstreetmap.org') + geocoords_from_map_link(location, 'openstreetmap.org', + session) if latitude and longitude: tags.append({ "@context": [ @@ -1745,7 +1746,8 @@ def _create_post_base(base_dir: str, languages_understood: [], translate: {}, buy_url: str, chat_url: str, auto_cw_cache: {}, - searchable_by: []) -> {}: + searchable_by: [], + session) -> {}: """Creates a message """ content = remove_invalid_chars(content) @@ -1863,7 +1865,8 @@ def _create_post_base(base_dir: str, _create_post_place_and_time(event_date, end_date, event_time, end_time, summary, content, schedule_post, - event_uuid, location, tags) + event_uuid, location, tags, + session) post_context = get_individual_post_context() @@ -2203,7 +2206,8 @@ def create_public_post(base_dir: str, languages_understood: [], translate: {}, buy_url: str, chat_url: str, auto_cw_cache: {}, - searchable_by: []) -> {}: + searchable_by: [], + session) -> {}: """Public post """ domain_full = get_full_domain(domain, port) @@ -2239,7 +2243,8 @@ def create_public_post(base_dir: str, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, - chat_url, auto_cw_cache, searchable_by) + chat_url, auto_cw_cache, searchable_by, + session) def create_reading_post(base_dir: str, @@ -2262,7 +2267,7 @@ def create_reading_post(base_dir: str, languages_understood: [], translate: {}, buy_url: str, chat_url: str, auto_cw_cache: {}, - searchable_by: []) -> {}: + searchable_by: [], session) -> {}: """ reading status post """ content = '' @@ -2303,7 +2308,7 @@ def create_reading_post(base_dir: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, auto_cw_cache, - searchable_by) + searchable_by, session) if post_json_object: post_json_object['object']['tag'] = [{ 'href': book_url, @@ -2363,7 +2368,7 @@ def create_blog_post(base_dir: str, media_license_url: str, media_creator: str, languages_understood: [], translate: {}, buy_url: str, chat_url: str, - searchable_by: []) -> {}: + searchable_by: [], session) -> {}: auto_cw_cache = {} blog_json = \ create_public_post(base_dir, @@ -2380,7 +2385,7 @@ def create_blog_post(base_dir: str, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) url_str = get_url_from_post(blog_json['object']['url']) obj_url = remove_html(url_str) if '/@/' not in obj_url: @@ -2400,7 +2405,7 @@ def create_news_post(base_dir: str, low_bandwidth: bool, content_license_url: str, media_license_url: str, media_creator: str, languages_understood: [], translate: {}, - buy_url: str, chat_url: str) -> {}: + buy_url: str, chat_url: str, session) -> {}: auto_cw_cache = {} client_to_server = False in_reply_to = None @@ -2426,7 +2431,7 @@ def create_news_post(base_dir: str, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) blog['object']['type'] = 'Article' return blog @@ -2444,7 +2449,7 @@ def create_question_post(base_dir: str, content_license_url: str, media_license_url: str, media_creator: str, languages_understood: [], translate: {}, - auto_cw_cache: {}) -> {}: + auto_cw_cache: {}, session) -> {}: """Question post with multiple choice options """ domain_full = get_full_domain(domain, port) @@ -2483,7 +2488,8 @@ def create_question_post(base_dir: str, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, - chat_url, auto_cw_cache, searchable_by) + chat_url, auto_cw_cache, searchable_by, + session) message_json['object']['type'] = 'Question' message_json['object']['oneOf'] = [] message_json['object']['votersCount'] = 0 @@ -2522,7 +2528,7 @@ def create_unlisted_post(base_dir: str, media_license_url: str, media_creator: str, languages_understood: [], translate: {}, buy_url: str, chat_url: str, - auto_cw_cache: {}) -> {}: + auto_cw_cache: {}, session) -> {}: """Unlisted post. This has the #Public and followers links inverted. """ domain_full = get_full_domain(domain, port) @@ -2559,7 +2565,7 @@ def create_unlisted_post(base_dir: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, auto_cw_cache, - searchable_by) + searchable_by, session) def create_followers_only_post(base_dir: str, @@ -2582,7 +2588,7 @@ def create_followers_only_post(base_dir: str, translate: {}, buy_url: str, chat_url: str, auto_cw_cache: {}, - searchable_by: []) -> {}: + searchable_by: [], session) -> {}: """Followers only post """ domain_full = get_full_domain(domain, port) @@ -2615,7 +2621,7 @@ def create_followers_only_post(base_dir: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, auto_cw_cache, - searchable_by) + searchable_by, session) def get_mentioned_people(base_dir: str, http_prefix: str, @@ -2675,7 +2681,7 @@ def create_direct_message_post(base_dir: str, languages_understood: [], dm_is_chat: bool, translate: {}, buy_url: str, chat_url: str, - auto_cw_cache: {}) -> {}: + auto_cw_cache: {}, session) -> {}: """Direct Message post """ content = resolve_petnames(base_dir, nickname, domain, content) @@ -2716,7 +2722,7 @@ def create_direct_message_post(base_dir: str, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) # mentioned recipients go into To rather than Cc message_json['to'] = message_json['object']['cc'] if not isinstance(message_json['to'], list): @@ -2744,7 +2750,7 @@ def create_report_post(base_dir: str, content_license_url: str, media_license_url: str, media_creator: str, languages_understood: [], translate: {}, - auto_cw_cache: {}) -> {}: + auto_cw_cache: {}, session) -> {}: """Send a report to moderators """ domain_full = get_full_domain(domain, port) @@ -2856,7 +2862,7 @@ def create_report_post(base_dir: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, auto_cw_cache, - searchable_by) + searchable_by, session) if not post_json_object: continue @@ -3169,7 +3175,7 @@ def send_post(signing_priv_key_pem: str, project_version: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) # get the senders private key private_key_pem = get_person_key(nickname, domain, base_dir, @@ -3391,7 +3397,7 @@ def send_post_via_server(signing_priv_key_pem: str, project_version: str, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, auto_cw_cache, - searchable_by) + searchable_by, session) auth_header = create_basic_auth_header(from_nickname, password) diff --git a/session.py b/session.py index 415a3fefb..07ab8a053 100644 --- a/session.py +++ b/session.py @@ -98,6 +98,31 @@ def url_exists(session, url: str, timeout_sec: int = 3, return False +def get_resolved_url(session, url: str, timeout_sec: int = 20) -> {}: + """returns the URL after redirections + eg. https://osm.org/go/0G0dJ91-?m=&relation=62414 + becomes + https://www.openstreetmap.org/?mlat=53.05289268493652 + &mlon=8.644180297851562#map=11/53.05289268493652/8.644180297851562 + """ + try: + result = session.get(url, headers={}, + params={}, timeout=timeout_sec, + allow_redirects=True) + if result.url: + if isinstance(result.url, str): + if '://' in result.url: + return result.url + except ValueError as exc: + print('EX: _get_resolved_url failed, url: ' + + str(url) + ', ' + str(exc)) + except SocketError as exc: + if exc.errno == errno.ECONNRESET: + print('EX: _get_resolved_url failed, ' + + 'connection was reset during _get_resolved_url ' + str(exc)) + return None + + def _get_json_request(session, url: str, session_headers: {}, session_params: {}, timeout_sec: int, quiet: bool, debug: bool, diff --git a/tests.py b/tests.py index 20e34d13a..69db47d46 100644 --- a/tests.py +++ b/tests.py @@ -809,6 +809,7 @@ def create_server_alice(path: str, domain: str, port: int, auto_cw_cache = {} test_video_transcript = '' searchable_by = [] + session = None create_public_post(path, nickname, domain, port, http_prefix, "No wise fish would go anywhere without a porpoise", test_save_to_file, @@ -827,7 +828,7 @@ def create_server_alice(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) create_public_post(path, nickname, domain, port, http_prefix, "Curiouser and curiouser!", test_save_to_file, @@ -846,7 +847,7 @@ def create_server_alice(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) create_public_post(path, nickname, domain, port, http_prefix, "In the gardens of memory, in the palace " + "of dreams, that is where you and I shall meet", @@ -866,7 +867,7 @@ def create_server_alice(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) regenerate_index_for_box(path, nickname, domain, 'outbox') global TEST_SERVER_ALICE_RUNNING TEST_SERVER_ALICE_RUNNING = True @@ -1005,6 +1006,7 @@ def create_server_bob(path: str, domain: str, port: int, auto_cw_cache = {} test_video_transcript = '' searchable_by = [] + session = None create_public_post(path, nickname, domain, port, http_prefix, "It's your life, live it your way.", test_save_to_file, @@ -1023,7 +1025,7 @@ def create_server_bob(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) create_public_post(path, nickname, domain, port, http_prefix, "One of the things I've realised is that " + "I am very simple", @@ -1043,7 +1045,7 @@ def create_server_bob(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) create_public_post(path, nickname, domain, port, http_prefix, "Quantum physics is a bit of a passion of mine", test_save_to_file, @@ -1062,7 +1064,7 @@ def create_server_bob(path: str, domain: str, port: int, low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) regenerate_index_for_box(path, nickname, domain, 'outbox') global TEST_SERVER_BOB_RUNNING TEST_SERVER_BOB_RUNNING = True @@ -3113,6 +3115,7 @@ def _test_create_person_account(base_dir: str): chat_url = '' auto_cw_cache = {} searchable_by = [] + session = None test_post_json = \ create_public_post(base_dir, nickname, domain, port, http_prefix, content, save_to_file, @@ -3128,7 +3131,7 @@ def _test_create_person_account(base_dir: str): low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) assert test_post_json assert test_post_json.get('object') assert test_post_json['object']['content'] @@ -3157,7 +3160,7 @@ def _test_create_person_account(base_dir: str): low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) assert test_post_json assert test_post_json.get('object') assert test_post_json['object']['content'] @@ -5018,6 +5021,7 @@ def _test_reply_to_public_post(base_dir: str) -> None: auto_cw_cache = {} video_transcript = '' searchable_by = [] + session = None reply = \ create_public_post(base_dir, nickname, domain, port, http_prefix, content, save_to_file, @@ -5034,7 +5038,7 @@ def _test_reply_to_public_post(base_dir: str) -> None: low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) # print(str(reply)) expected_str = \ '

' + \ @@ -6064,6 +6068,7 @@ def _test_links_within_post(base_dir: str) -> None: auto_cw_cache = {} video_transcript = '' searchable_by = [] + session = None post_json_object = \ create_public_post(base_dir, nickname, domain, port, http_prefix, @@ -6080,7 +6085,7 @@ def _test_links_within_post(base_dir: str) -> None: low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) expected_str = \ '

This is a test post with links.

' + \ @@ -6127,7 +6132,7 @@ def _test_links_within_post(base_dir: str) -> None: low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) assert post_json_object['object']['content'] == content assert post_json_object['object']['contentMap'][system_language] == content @@ -6152,7 +6157,7 @@ def _test_links_within_post(base_dir: str) -> None: low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) if post_json_object['object']['content'] != content: print('content1: ' + post_json_object['object']['content']) print('content2: ' + content) @@ -7255,6 +7260,7 @@ def _test_can_replyto(base_dir: str) -> None: auto_cw_cache = {} video_transcript = '' searchable_by = [] + session = None post_json_object = \ create_public_post(base_dir, nickname, domain, port, http_prefix, @@ -7271,7 +7277,7 @@ def _test_can_replyto(base_dir: str) -> None: low_bandwidth, content_license_url, media_license_url, media_creator, languages_understood, translate, buy_url, chat_url, - auto_cw_cache, searchable_by) + auto_cw_cache, searchable_by, session) # set the date on the post curr_date_str = "2021-09-08T20:45:00Z" post_json_object['published'] = curr_date_str @@ -8096,7 +8102,8 @@ def _test_hashtag_maps(): "

This is a test, with a geo link " + \ "geo:52.90820,-3.59817;u=35, and some other stuff," + \ " with commas

" - map_links = get_map_links_from_post_content(content) + session = None + map_links = get_map_links_from_post_content(content, session) link = "geo:52.90820,-3.59817" if link not in map_links: print('map_links: ' + str(map_links)) @@ -8125,11 +8132,12 @@ def _test_hashtag_maps(): "epicyon.libreserver.org/tags/AnotherHashtag\" " + \ "class=\"mention hashtag\" rel=\"tag\" tabindex=\"10\">#" + \ "AnotherHashtag

" - map_links = get_map_links_from_post_content(content) + map_links = get_map_links_from_post_content(content, session) link = "www.google.com/maps/@52.217291,-3.0811865,20.04z" assert link in map_links + session = None zoom, latitude, longitude = \ - geocoords_from_map_link(link, 'openstreetmap.org') + geocoords_from_map_link(link, 'openstreetmap.org', session) assert zoom == 20 assert latitude assert int(latitude * 1000) == 52217 @@ -8138,7 +8146,7 @@ def _test_hashtag_maps(): link = "www.openstreetmap.org/#map=19/52.90860/-3.59917" assert link in map_links zoom, latitude, longitude = \ - geocoords_from_map_link(link, 'openstreetmap.org') + geocoords_from_map_link(link, 'openstreetmap.org', session) assert zoom == 19 assert latitude assert int(latitude * 1000) == 52908 diff --git a/webapp_calendar.py b/webapp_calendar.py index b54e9c4b1..51e10212f 100644 --- a/webapp_calendar.py +++ b/webapp_calendar.py @@ -117,7 +117,8 @@ def _html_calendar_day(person_cache: {}, translate: {}, nickname: str, domain: str, day_events: [], month_name: str, actor: str, theme: str, access_keys: {}, - system_language: str) -> str: + system_language: str, + session, session_onion, session_i2p) -> str: """Show a day within the calendar """ account_dir = acct_dir(base_dir, nickname, domain) @@ -232,7 +233,9 @@ def _html_calendar_day(person_cache: {}, translate: {}, event_map = \ html_open_street_map(event_place, bounding_box_degrees, - translate, + translate, session, + session_onion, + session_i2p, '320', '320') if event_map: event_place = event_map @@ -354,7 +357,8 @@ def html_calendar(person_cache: {}, translate: {}, http_prefix: str, domain_full: str, text_mode_banner: str, access_keys: {}, icalendar: bool, system_language: str, - default_timeline: str, theme: str) -> str: + default_timeline: str, theme: str, + session, session_onion, session_i2p) -> str: """Show the calendar for a person """ domain = remove_domain_port(domain_full) @@ -441,7 +445,8 @@ def html_calendar(person_cache: {}, translate: {}, nickname, domain, day_events, month_name, actor, theme, access_keys, - system_language) + system_language, session, + session_onion, session_i2p) if icalendar: return get_month_events_icalendar(base_dir, nickname, domain, diff --git a/webapp_hashtagswarm.py b/webapp_hashtagswarm.py index a5cc9feca..7542fac84 100644 --- a/webapp_hashtagswarm.py +++ b/webapp_hashtagswarm.py @@ -338,7 +338,8 @@ def _update_cached_hashtag_swarm(base_dir: str, nickname: str, domain: str, def store_hash_tags(base_dir: str, nickname: str, domain: str, http_prefix: str, domain_full: str, - post_json_object: {}, translate: {}) -> None: + post_json_object: {}, translate: {}, + session) -> None: """Extracts hashtags from an incoming post and updates the relevant tags files. """ @@ -366,14 +367,14 @@ def store_hash_tags(base_dir: str, nickname: str, domain: str, if 'content' in post_json_object['object']: published = post_json_object['object']['published'] post_content = post_json_object['object']['content'] - map_links += get_map_links_from_post_content(post_content) + map_links += get_map_links_from_post_content(post_content, session) # get geolocation from tags location_str = get_location_from_post(post_json_object) if location_str: if resembles_url(location_str): zoom, latitude, longitude = \ geocoords_from_map_link(location_str, - 'openstreetmap.org') + 'openstreetmap.org', session) if latitude and longitude and zoom and \ location_str not in map_links: map_links.append(location_str) diff --git a/webapp_post.py b/webapp_post.py index fcb4d5391..425913906 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -2982,7 +2982,8 @@ def individual_post_as_html(signing_priv_key_pem: str, map_str = \ html_open_street_map(location_str, bounding_box_degrees, - translate) + translate, session, + session, session) if map_str: map_str = '
\n' + map_str + '
\n' attrib = None @@ -3009,7 +3010,8 @@ def individual_post_as_html(signing_priv_key_pem: str, # remember the coordinates map_zoom, map_latitude, map_longitude = \ geocoords_from_map_link(location_str, - 'openstreetmap.org') + 'openstreetmap.org', + session) if map_zoom and map_latitude and map_longitude: set_map_preferences_coords(base_dir, nickname, domain, map_latitude, map_longitude, diff --git a/webapp_search.py b/webapp_search.py index df68e7dc0..315a3098b 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -953,7 +953,7 @@ def html_hashtag_search(nickname: str, domain: str, port: int, # maps for geolocations with this hashtag maps_str = html_hashtag_maps(base_dir, hashtag, translate, map_format, - nickname, domain) + nickname, domain, session) if maps_str: maps_str = '
' + maps_str + '
\n' hashtag_search_form += maps_str