From e88bc0085b7412fb916db2822d2692c3ec497ff5 Mon Sep 17 00:00:00 2001 From: bashrc Date: Sun, 23 Aug 2026 19:40:41 +0100 Subject: [PATCH] Create indexes for actor labels --- src/content_labels.py | 122 +++++++++++++++++++++++++++++++++---- src/daemon_post_profile.py | 15 +++-- src/delete.py | 33 ---------- src/inbox_receive.py | 17 ++++-- src/posts.py | 16 ++++- 5 files changed, 148 insertions(+), 55 deletions(-) diff --git a/src/content_labels.py b/src/content_labels.py index 82c265222..10527a2c9 100644 --- a/src/content_labels.py +++ b/src/content_labels.py @@ -25,6 +25,7 @@ from src.data import is_a_dir from src.data import makedir from src.data import is_a_file from src.data import save_string +from src.data import erase_file from src.maps import get_map_links_from_post_content from src.maps import get_location_from_post from src.maps import geocoords_from_map_link @@ -32,11 +33,44 @@ from src.maps import add_label_map_links from src.timeFunctions import date_utcnow from src.timeFunctions import date_epoch from src.timeFunctions import date_from_string_format -# from src.delete import remove_old_labels +from src.timeFunctions import date_from_numbers MAX_CONTENT_LABELS = 10 +def _remove_old_labels(base_dir: str, max_months: int) -> str: + """Remove old labels + """ + max_months: int = min(max_months, 11) + prev_date = date_from_numbers(1970, 1 + max_months, 1, 0, 0) + max_days_since_epoch: int = (date_utcnow() - prev_date).days + remove_labels: list[str] = [] + + for _, _, files in os.walk(base_dir + '/labels'): + for fname in files: + labels_filename: str = os.path.join(base_dir + '/labels', fname) + if not is_a_file(labels_filename): + continue + # get last modified datetime + mod_time_since_epoc = os.path.getmtime(labels_filename) + last_modified_date = \ + datetime.fromtimestamp(mod_time_since_epoc, + timezone.utc) + prev_date_epoch = date_epoch() + file_days_since_epoch = \ + (last_modified_date - prev_date_epoch).days + + # check of the file is too old + if file_days_since_epoch < max_days_since_epoch: + remove_labels.append(labels_filename) + break + + for erase_filename in remove_labels: + erase_file(erase_filename, + 'EX: remove_old_labels unable to delete ' + + erase_filename) + + def get_labels_from_json(json_object: {}) -> []: """Returns labels attached to an actor or post """ @@ -227,7 +261,7 @@ def _store_content_label(nickname: str, """ if not valid_content_label(label): return False - labels_filename = labels_dir + '/' + label + '.txt' + labels_filename: str = labels_dir + '/' + label + '.txt' days_diff = date_utcnow() - date_epoch() days_since_epoch = days_diff.days label_line = \ @@ -260,6 +294,38 @@ def _store_content_label(nickname: str, return True +def _store_person_label(label: str, labels_dir: str, actor_url: str) -> bool: + """stores an individual label for a person + """ + if not valid_content_label(label): + return False + actor_url: str = remove_id_ending(actor_url) + labels_filename: str = labels_dir + '/' + label + '.txt' + + label_added: bool = False + if not is_a_file(labels_filename): + if save_string(actor_url, labels_filename, + 'EX: _store_person_label unable to write ' + + labels_filename + ' [ex]'): + label_added = True + else: + content: str = load_string(labels_filename, + 'EX: _store_person_label failed to read ' + + labels_filename + ' [ex]') + if content is None: + content = '' + if actor_url + '\n' not in content: + content = actor_url + '\n' + content + if save_string(content, labels_filename, + 'EX: Failed to write entry to labels file ' + + labels_filename + ' [ex]'): + label_added = True + + if not label_added: + return False + return False + + def _html_labels_swarm(base_dir: str, actor: str) -> str: """Returns a labels swarm of today's labels """ @@ -412,6 +478,7 @@ def _update_cached_labels_swarm(base_dir: str, nickname: str, domain: str, else: print('WARN: no modified date for ' + str(last_modified)) if save_swarm: + _remove_old_labels(base_dir, 3) actor = local_actor_url(http_prefix, nickname, domain_full) new_swarm_str = _html_labels_swarm(base_dir, actor) if new_swarm_str: @@ -419,14 +486,13 @@ def _update_cached_labels_swarm(base_dir: str, nickname: str, domain: str, 'EX: unable to write cached labels swarm ' + cached_labels_swarm_filename): return True - # remove_old_labels(base_dir, 3) return False -def store_content_labels(base_dir: str, nickname: str, domain: str, - http_prefix: str, domain_full: str, - post_json_object: {}, - session) -> None: +def _store_content_labels_base(base_dir: str, nickname: str, domain: str, + http_prefix: str, domain_full: str, + post_json_object: {}, + session, is_person: bool) -> None: """Extracts content labels from an incoming post and updates the relevant label files. """ @@ -435,6 +501,8 @@ def store_content_labels(base_dir: str, nickname: str, domain: str, return labels_dir = base_dir + '/labels' + if is_person: + labels_dir = base_dir + '/personlabels' # add labels directory if it doesn't exist if not is_a_dir(labels_dir): @@ -472,15 +540,43 @@ def store_content_labels(base_dir: str, nickname: str, domain: str, post_url = remove_id_ending(post_json_object['id']) post_url = post_url.replace('/', '#') labels_ctr: int = 0 - for label in labels_list: - if _store_content_label(nickname, - label, labels_dir, post_url, - map_links, published, - labels_maps_dir): - labels_ctr += 1 + if is_person: + for label in labels_list: + if _store_content_label(nickname, + label, labels_dir, post_url, + map_links, published, + labels_maps_dir): + labels_ctr += 1 + else: + for label in labels_list: + if _store_person_label(label, labels_dir, post_url): + labels_ctr += 1 # if some labels were found then recalculate the swarm # ready for later display if labels_ctr > 0: _update_cached_labels_swarm(base_dir, nickname, domain, http_prefix, domain_full) + + +def store_person_labels(base_dir: str, nickname: str, domain: str, + http_prefix: str, domain_full: str, + actor_json: {}, session) -> None: + """Extracts labels from an incoming actor to the + relevant label files. + """ + _store_content_labels_base(base_dir, nickname, domain, + http_prefix, domain_full, + actor_json, session, True) + + +def store_content_labels(base_dir: str, nickname: str, domain: str, + http_prefix: str, domain_full: str, + post_json_object: {}, + session) -> None: + """Extracts labels from an incoming post to the + relevant label files. + """ + _store_content_labels_base(base_dir, nickname, domain, + http_prefix, domain_full, + post_json_object, session, False) diff --git a/src/daemon_post_profile.py b/src/daemon_post_profile.py index 1f593bfad..374976f9a 100644 --- a/src/daemon_post_profile.py +++ b/src/daemon_post_profile.py @@ -159,6 +159,7 @@ from src.data import is_a_dir from src.data import makedir from src.content_labels import get_actor_content_labels from src.content_labels import set_actor_content_labels +from src.content_labels import store_person_labels def _profile_post_deactivate_account(base_dir: str, nickname: str, domain: str, @@ -177,7 +178,8 @@ def _profile_post_deactivate_account(base_dir: str, nickname: str, domain: str, def _profile_post_save_actor(base_dir: str, http_prefix: str, - nickname: str, domain: str, port: int, + nickname: str, domain: str, + domain_full: str, port: int, actor_json: {}, actor_filename: str, onion_domain: str, i2p_domain: str, yggdrasil_domain: str, @@ -185,7 +187,8 @@ def _profile_post_save_actor(base_dir: str, http_prefix: str, send_move_activity: bool, self, cached_webfingers: {}, person_cache: {}, project_version: str, - system_language: str) -> None: + system_language: str, + session) -> None: """ HTTP POST save actor json file within accounts """ add_name_emojis_to_tags(base_dir, http_prefix, @@ -210,6 +213,10 @@ def _profile_post_save_actor(base_dir: str, http_prefix: str, webfinger_update(base_dir, nickname, domain, onion_domain, i2p_domain, yggdrasil_domain, cached_webfingers) + # store any labels for the person + store_person_labels(base_dir, nickname, domain, + http_prefix, domain_full, + actor_json, session) # also copy to the actors cache and # person_cache in memory store_person_in_cache(base_dir, actor_json['id'], actor_json, @@ -3461,7 +3468,7 @@ def profile_edit(self, calling_domain: str, cookie: str, # save actor json file within accounts if actor_changed: _profile_post_save_actor(base_dir, http_prefix, - nickname, domain, + nickname, domain, domain_full, self.server.port, actor_json, actor_filename, onion_domain, i2p_domain, @@ -3470,7 +3477,7 @@ def profile_edit(self, calling_domain: str, cookie: str, send_move_activity, self, cached_webfingers, person_cache, project_version, - system_language) + system_language, curr_session) if _profile_post_deactivate_account(base_dir, nickname, domain, calling_domain, diff --git a/src/delete.py b/src/delete.py index 33d845e8d..74c032c94 100644 --- a/src/delete.py +++ b/src/delete.py @@ -219,36 +219,3 @@ def remove_old_hashtags(base_dir: str, max_months: int) -> str: erase_file(erase_filename, 'EX: remove_old_hashtags unable to delete ' + erase_filename) - - -def remove_old_labels(base_dir: str, max_months: int) -> str: - """Remove old labels - """ - max_months: int = min(max_months, 11) - prev_date = date_from_numbers(1970, 1 + max_months, 1, 0, 0) - max_days_since_epoch: int = (date_utcnow() - prev_date).days - remove_labels: list[str] = [] - - for _, _, files in os.walk(base_dir + '/labels'): - for fname in files: - labels_filename: str = os.path.join(base_dir + '/labels', fname) - if not is_a_file(labels_filename): - continue - # get last modified datetime - mod_time_since_epoc = os.path.getmtime(labels_filename) - last_modified_date = \ - datetime.fromtimestamp(mod_time_since_epoc, - timezone.utc) - prev_date_epoch = date_epoch() - file_days_since_epoch = \ - (last_modified_date - prev_date_epoch).days - - # check of the file is too old - if file_days_since_epoch < max_days_since_epoch: - remove_labels.append(labels_filename) - break - - for erase_filename in remove_labels: - erase_file(erase_filename, - 'EX: remove_old_labels unable to delete ' + - erase_filename) diff --git a/src/inbox_receive.py b/src/inbox_receive.py index 5081eccd5..a32d62b9a 100644 --- a/src/inbox_receive.py +++ b/src/inbox_receive.py @@ -102,6 +102,7 @@ from src.acceptreject import create_feature_accept from src.posts import send_signed_json from src.collections import store_feature_authorization from src.collections import allow_lists +from src.content_labels import store_person_labels def inbox_update_index(boxname: str, base_dir: str, handle: str, @@ -183,11 +184,12 @@ def _notify_moved(base_dir: str, domain_full: str, def _person_receive_update(base_dir: str, - domain: str, port: int, + nickname: str, domain: str, port: int, update_nickname: str, update_domain: str, update_port: int, person_json: {}, person_cache: {}, - debug: bool, http_prefix: str) -> bool: + debug: bool, http_prefix: str, + session) -> bool: """Changes an actor. eg: avatar or display name change """ url_str = get_url_from_post(person_json['url']) @@ -248,6 +250,12 @@ def _person_receive_update(base_dir: str, print('WARN: Public key does not match ' + 'cached actor when updating') return False + + # store any labels for the person + store_person_labels(base_dir, nickname, domain, + http_prefix, domain_full, + person_json, session) + # save to cache in memory store_person_in_cache(base_dir, idx, person_json, person_cache, True) @@ -773,11 +781,12 @@ def receive_update_activity(recent_posts_cache: {}, session, base_dir: str, get_domain_from_actor(actor_url) if update_nickname and update_domain: if _person_receive_update(base_dir, - domain, port, + nickname, domain, port, update_nickname, update_domain, update_port, message_json['object'], - person_cache, debug, http_prefix): + person_cache, debug, http_prefix, + session): print('Person Update: ' + str(message_json)) if debug: print('DEBUG: Profile update was received for ' + diff --git a/src/posts.py b/src/posts.py index 73faef650..e70c6d016 100644 --- a/src/posts.py +++ b/src/posts.py @@ -156,6 +156,7 @@ from src.data import is_a_file from src.data import is_a_dir from src.data import makedir from src.content_labels import set_post_content_labels +from src.content_labels import store_person_labels def convert_post_content_to_html(message_json: {}) -> None: @@ -427,7 +428,8 @@ def get_person_box(signing_priv_key_pem: str, origin_domain: str, person_id = None if person_json.get('id'): - person_id = person_json['id'] + if isinstance(person_json['id'], str): + person_id = person_json['id'] pub_key, pub_key_id = get_actor_public_key_from_id(person_json, None) shared_inbox = None if person_json.get('sharedInbox'): @@ -463,6 +465,18 @@ def get_person_box(signing_priv_key_pem: str, origin_domain: str, if person_json.get('movedTo') or person_json.get('copiedTo'): display_name += ' ⌂' + # store any labels for the person + if person_id: + person_http_prefix = http_prefix + if '://' in person_id: + person_http_prefix = http_prefix.split('://')[0] + person_nickname = get_nickname_from_actor(person_id) + person_domain, person_port = get_domain_from_actor(person_id) + person_domain_full = get_full_domain(person_domain, person_port) + store_person_labels(base_dir, person_nickname, person_domain, + person_http_prefix, person_domain_full, + person_json, session) + store_person_in_cache(base_dir, person_url, person_json, person_cache, True)