From c8e9e37bcef41b59a8be6baedb224454e6f6a893 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 22 Aug 2022 16:39:24 +0100 Subject: [PATCH] Hashtag maps --- daemon.py | 20 ++++++ maps.py | 155 +++++++++++++++++++++++++++++++++++++++++++++++ webapp_search.py | 3 + 3 files changed, 178 insertions(+) diff --git a/daemon.py b/daemon.py index 684cdb2e5..671e2caec 100644 --- a/daemon.py +++ b/daemon.py @@ -403,6 +403,7 @@ from crawlers import blocked_user_agent from crawlers import load_known_web_bots from qrcode import save_domain_qrcode from importFollowing import run_import_following_watchdog +from maps import kml_from_tagmaps_path import os @@ -14243,6 +14244,7 @@ class PubServer(BaseHTTPRequestHandler): '/statuses/' not in path and \ '/emoji/' not in path and \ '/tags/' not in path and \ + '/tagmaps/' not in path and \ '/avatars/' not in path and \ '/favicons/' not in path and \ '/headers/' not in path and \ @@ -17086,6 +17088,24 @@ class PubServer(BaseHTTPRequestHandler): self.server.getreq_busy = False return + # hashtag map kml + if self.path.startswith('/tagmaps/') or \ + (authorized and '/tagmaps/' in self.path): + kml_str = kml_from_tagmaps_path(self.server.base_dir, self.path) + if kml_str: + msg = kml_str.encode('utf-8') + msglen = len(msg) + header_type = \ + 'application/vnd.google-earth.kml+xml; charset=utf-8' + self._set_headers(header_type, msglen, + None, calling_domain, True) + self._write(msg) + self.server.getreq_busy = False + return + self._404() + self.server.getreq_busy = False + return + fitness_performance(getreq_start_time, self.server.fitness, '_GET', 'hashtag search done', self.server.debug) diff --git a/maps.py b/maps.py index 2b5ff2885..8c98b67cf 100644 --- a/maps.py +++ b/maps.py @@ -457,3 +457,158 @@ def add_tag_map_links(tag_maps_dir: str, tag_name: str, fp_tag.write(map_links_str) except OSError: print('EX: error writing tag map ' + tag_map_filename) + + +def _hashtag_map_to_kml(base_dir: str, tag_name: str, + start_hours_since_epoch: int, + end_hours_since_epoch: int) -> str: + """Returns the KML for a given hashtag between the given times + """ + place_ctr = 0 + osm_domain = 'openstreetmap.org' + tag_map_filename = base_dir + '/tagmaps/' + tag_name + '.txt' + + kml_str = '\n' + kml_str += '\n' + kml_str += '\n' + + if os.path.isfile(tag_map_filename): + map_links = [] + try: + with open(tag_map_filename, 'r', encoding='utf-8') as fp_tag: + map_links = fp_tag.readlines() + except OSError: + print('EX: unable to read tag map links ' + tag_map_filename) + if map_links: + start_secs_since_epoch = start_hours_since_epoch * 60 * 60 + end_secs_since_epoch = end_hours_since_epoch * 60 * 60 + for link_line in map_links: + link_line = link_line.strip().split(' ') + if len(link_line) < 3: + continue + secs_since_epoch = int(link_line[0]) + if secs_since_epoch < start_secs_since_epoch or \ + secs_since_epoch > end_secs_since_epoch: + continue + map_link = link_line[1] + zoom, latitude, longitude = \ + geocoords_from_map_link(map_link, osm_domain) + if not zoom: + continue + if not latitude: + continue + if not longitude: + continue + post_id = link_line[2] + place_ctr += 1 + kml_str += '\n' + kml_str += ' ' + str(place_ctr) + '\n' + kml_str += ' ' + \ + post_id + '\n]]\n' + kml_str += ' \n' + kml_str += ' \n' + kml_str += ' ' + str(longitude) + ',' + \ + str(latitude) + ',0\n' + kml_str += ' \n' + kml_str += '\n' + + kml_str += '\n' + kml_str += '' + if place_ctr == 0: + return None + return kml_str + + +def _hashtag_map_kml_within_hours(base_dir: str, tag_name: str, + hours: int) -> str: + """Returns kml for a hashtag containing maps for the last number of hours + """ + secs_since_epoch = \ + (datetime.datetime.utcnow() - + datetime.datetime(1970, 1, 1)).total_seconds() + end_hours_since_epoch = int(secs_since_epoch / (60 * 60)) + start_hours_since_epoch = end_hours_since_epoch - hours + kml_str = \ + _hashtag_map_to_kml(base_dir, tag_name, + start_hours_since_epoch, + end_hours_since_epoch) + return kml_str + + +def _get_tagmaps_time_periods() -> {}: + """dict of time periods for map display + """ + return { + "Last hour": -1, + "Last 3 hours": -3, + "Last 6 hours": -6, + "Last 12 hours": -12, + "Last day": -24, + "Last two days": -48, + "Last week": -24 * 7, + "Last two weeks": -24 * 7 * 2, + "Last month": -24 * 7 * 4, + "Last six months": -24 * 7 * 4 * 6, + "Last year": -24 * 7 * 4 * 12 + } + + +def kml_from_tagmaps_path(base_dir: str, path: str) -> str: + """Returns kml for a given tagmaps path + /tagmaps/tagname-time_period + """ + if '/tagmaps/' not in path: + return None + time_period = _get_tagmaps_time_periods() + tag_name = path.split('/tagmaps/')[1] + if '-' in tag_name: + tag_name = tag_name.split('-')[0] + if not tag_name: + return None + for period_str, hours in time_period.items(): + period_str2 = period_str.replace('Last ', '').lower() + endpoint_str = \ + '/tagmaps/' + tag_name + '-' + period_str2.replace(' ', '_') + if path == endpoint_str: + return _hashtag_map_kml_within_hours(base_dir, tag_name, + hours) + return None + + +def html_hashtag_maps(base_dir: str, tag_name: str, + translate: {}) -> str: + """Returns html for maps associated with a hashtag + """ + tag_map_filename = base_dir + '/tagmaps/' + tag_name + '.txt' + if not os.path.isfile(tag_map_filename): + return '' + + time_period = _get_tagmaps_time_periods() + + html_str = '' + kml_str = None + for period_str, hours in time_period.items(): + new_kml_str = \ + _hashtag_map_kml_within_hours(base_dir, tag_name, hours) + if not new_kml_str: + continue + if new_kml_str != kml_str: + kml_str = new_kml_str + period_str2 = period_str.replace('Last ', '').lower() + endpoint_str = \ + '/tagmaps/' + tag_name + '-' + period_str2.replace(' ', '_') + download_filename = \ + (tag_name + '-' + + period_str.lower()).replace(' ', '_') + '.kml' + if html_str: + html_str += ' ' + description = period_str + if translate.get(period_str): + description = translate[period_str] + html_str += '' + \ + description + '' + if html_str: + html_str = '
' + html_str + '
\n' + return html_str diff --git a/webapp_search.py b/webapp_search.py index c30b78e97..333471058 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -40,6 +40,7 @@ from webapp_utils import html_post_separator from webapp_utils import html_search_result_share from webapp_post import individual_post_as_html from webapp_hashtagswarm import html_hash_tag_swarm +from maps import html_hashtag_maps def html_search_emoji(translate: {}, base_dir: str, search_str: str) -> str: @@ -824,6 +825,8 @@ def html_hashtag_search(nickname: str, domain: str, port: int, hashtag_search_form += ' \n' hashtag_search_form += '\n' + hashtag_search_form += html_hashtag_maps(base_dir, hashtag, translate) + if start_index > 0: # previous page link hashtag_search_form += \