diff --git a/daemon.py b/daemon.py index 6876bcfd9..6130c8589 100644 --- a/daemon.py +++ b/daemon.py @@ -22114,6 +22114,7 @@ def run_daemon(max_hashtags: int, httpd.maxMessageLength = 64000 # Maximum overall number of posts per box httpd.maxPostsInBox = 32000 + httpd.maxCacheAgeDays = 30 httpd.domain = domain httpd.port = port httpd.domain_full = get_full_domain(domain, port) @@ -22259,7 +22260,8 @@ def run_daemon(max_hashtags: int, httpd.http_prefix, archive_dir, httpd.recent_posts_cache, - httpd.maxPostsInBox), daemon=True) + httpd.maxPostsInBox, + httpd.maxCacheAgeDays), daemon=True) begin_thread(httpd.thrCache, 'run_daemon thrCache') # number of mins after which sending posts or updates will expire diff --git a/epicyon.py b/epicyon.py index 7529345a2..6f078742d 100644 --- a/epicyon.py +++ b/epicyon.py @@ -585,6 +585,9 @@ def _command_options() -> None: parser.add_argument('--maxposts', dest='archiveMaxPosts', type=int, default=32000, help='Maximum number of posts in in/outbox') + parser.add_argument('--maxCacheAgeDays', dest='maxCacheAgeDays', type=int, + default=30, + help='Maximum age of files in the announce cache') parser.add_argument('--minimumvotes', dest='minimumvotes', type=int, default=1, help='Minimum number of votes to remove or add' + @@ -2832,7 +2835,8 @@ def _command_options() -> None: # archiving is for non-instance posts archive_media(base_dir, argb.archive, argb.archiveWeeks) archive_posts(base_dir, http_prefix, argb.archive, {}, - argb.archiveMaxPosts) + argb.archiveMaxPosts, + argb.maxCacheAgeDays) print('Archiving complete') # expiry is for instance posts, where an expiry period # has been set within profile settings diff --git a/importFollowing.py b/importFollowing.py index de74ae990..d8b4b4d86 100644 --- a/importFollowing.py +++ b/importFollowing.py @@ -64,9 +64,10 @@ def _update_import_following(base_dir: str, if ',' in line: fields = line.split(',') line = fields[0].strip() - if len(fields) >= 3: - notes = fields[2] + if len(fields) >= 5: + notes = fields[4] if line.startswith('#'): + # comment continue following_nickname = get_nickname_from_actor(line) if not following_nickname: diff --git a/posts.py b/posts.py index 8ad328394..b5b1e592a 100644 --- a/posts.py +++ b/posts.py @@ -4265,7 +4265,8 @@ def _create_box_indexed(recent_posts_cache: {}, def expire_cache(base_dir: str, person_cache: {}, http_prefix: str, archive_dir: str, recent_posts_cache: {}, - max_posts_in_box: int): + max_posts_in_box: int, + max_cache_age_days: int): """Thread used to expire actors from the cache and archive old posts """ while True: @@ -4273,12 +4274,42 @@ def expire_cache(base_dir: str, person_cache: {}, time.sleep(60 * 60 * 24) expire_person_cache(person_cache) archive_posts(base_dir, http_prefix, archive_dir, recent_posts_cache, - max_posts_in_box) + max_posts_in_box, max_cache_age_days) + + +def _expire_announce_cache_for_person(base_dir: str, + nickname: str, domain: str, + max_age_days: int) -> int: + """Expires entries within the announces cache + """ + cache_dir = base_dir + '/cache/announce/' + nickname + if not os.path.isdir(cache_dir): + print('No cached announces for ' + nickname + '@' + domain) + return 0 + expired_post_count = 0 + posts_in_cache = os.scandir(cache_dir) + for cache_filename in posts_in_cache: + cache_filename = cache_filename.name + # Time of file creation + full_filename = os.path.join(cache_dir, cache_filename) + if not os.path.isfile(full_filename): + continue + last_modified = file_last_modified(full_filename) + # get time difference + if not valid_post_date(last_modified, max_age_days, False): + try: + os.remove(full_filename) + except OSError: + print('EX: unable to delete from announce cache ' + + full_filename) + expired_post_count += 1 + return expired_post_count def archive_posts(base_dir: str, http_prefix: str, archive_dir: str, recent_posts_cache: {}, - max_posts_in_box=32000) -> None: + max_posts_in_box: int = 32000, + max_cache_age_days: int = 30) -> None: """Archives posts for all accounts """ if max_posts_in_box == 0: @@ -4315,6 +4346,12 @@ def archive_posts(base_dir: str, http_prefix: str, archive_dir: str, nickname, domain, base_dir, 'inbox', archive_subdir, recent_posts_cache, max_posts_in_box) + expired_announces = \ + _expire_announce_cache_for_person(base_dir, + nickname, domain, + max_cache_age_days) + print('Expired ' + str(expired_announces) + + ' cached announces for ' + nickname + '@' + domain) if archive_dir: archive_subdir = archive_dir + '/accounts/' + \ handle + '/outbox' diff --git a/webapp_utils.py b/webapp_utils.py index b9ae21dae..24b1ea0ed 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -125,10 +125,9 @@ def csv_following_list(following_filename: str, allowed_announce(base_dir, nickname, domain, following_nickname, following_domain) - - following_list_csv += \ - following_address + ',' + \ - str(announce_is_allowed).lower() + ',' + notify_on_new = 'false' + languages = '' + person_notes = '' person_notes_filename = \ acct_dir(base_dir, nickname, domain) + \ '/notes/' + following_address + '.txt' @@ -140,8 +139,15 @@ def csv_following_list(following_filename: str, person_notes = person_notes.replace('"', "'") person_notes = person_notes.replace('\n', '
') person_notes = person_notes.replace(' ', ' ') - following_list_csv += person_notes - msg = 'Account address,Show boosts,Notes\n' + following_list_csv + following_list_csv += \ + 'Account address,Show boosts,' + \ + 'Notify on new posts,Languages,Notes\n' + \ + following_address + ',' + \ + str(announce_is_allowed).lower() + ',' + \ + notify_on_new + ',' + \ + languages + ',' + \ + person_notes + '\n' + msg = following_list_csv return msg return ''