diff --git a/blog.py b/blog.py index d60e73b69..1cdb1d12f 100644 --- a/blog.py +++ b/blog.py @@ -17,6 +17,7 @@ from webapp_utils import html_footer from webapp_utils import get_post_attachments_as_html from webapp_utils import edit_text_area from webapp_media import add_embedded_elements +from utils import text_in_file from utils import local_actor_url from utils import get_actor_languages_list from utils import get_base_content_from_post @@ -929,8 +930,7 @@ def path_contains_blog_link(base_dir: str, acct_dir(base_dir, nickname, domain) + '/tlblogs.index' if not os.path.isfile(blog_index_filename): return None, None - if '#' + user_ending2[1] + '.' not in open(blog_index_filename, - encoding='utf-8').read(): + if not text_in_file('#' + user_ending2[1] + '.', blog_index_filename): return None, None message_id = local_actor_url(http_prefix, nickname, domain_full) + \ '/statuses/' + user_ending2[1] diff --git a/bookmarks.py b/bookmarks.py index 307b544f6..dbb31116f 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -28,6 +28,7 @@ from utils import acct_dir from utils import local_actor_url from utils import has_actor from utils import has_object_string_type +from utils import text_in_file from posts import get_person_box from session import post_json @@ -71,8 +72,7 @@ def undo_bookmarks_collection_entry(recent_posts_cache: {}, else: bookmark_index = post_filename.strip() bookmark_index = bookmark_index.replace('\n', '').replace('\r', '') - if bookmark_index not in open(bookmarks_index_filename, - encoding='utf-8').read(): + if not text_in_file(bookmark_index, bookmarks_index_filename): return index_str = '' try: @@ -238,8 +238,7 @@ def update_bookmarks_collection(recent_posts_cache: {}, acct_dir(base_dir, nickname, domain) + '/bookmarks.index' bookmark_index = post_filename.split('/')[-1] if os.path.isfile(bookmarks_index_filename): - if bookmark_index not in open(bookmarks_index_filename, - encoding='utf-8').read(): + if not text_in_file(bookmark_index, bookmarks_index_filename): try: with open(bookmarks_index_filename, 'r+', encoding='utf-8') as bmi_file: diff --git a/conversation.py b/conversation.py index 8b1e697e6..784d991d8 100644 --- a/conversation.py +++ b/conversation.py @@ -11,6 +11,7 @@ import os from utils import has_object_dict from utils import acct_dir from utils import remove_id_ending +from utils import text_in_file def _get_conversation_filename(base_dir: str, nickname: str, domain: str, @@ -50,8 +51,7 @@ def update_conversation(base_dir: str, nickname: str, domain: str, except OSError: print('EX: update_conversation ' + 'unable to write to ' + conversation_filename) - elif post_id + '\n' not in open(conversation_filename, - encoding='utf-8').read(): + elif not text_in_file(post_id + '\n', conversation_filename): try: with open(conversation_filename, 'a+', encoding='utf-8') as conv_file: diff --git a/filters.py b/filters.py index 97ca31432..226aff75d 100644 --- a/filters.py +++ b/filters.py @@ -9,6 +9,7 @@ __module_group__ = "Moderation" import os from utils import acct_dir +from utils import text_in_file def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool: @@ -54,7 +55,7 @@ def remove_filter(base_dir: str, nickname: str, domain: str, filters_filename = acct_dir(base_dir, nickname, domain) + '/filters.txt' if not os.path.isfile(filters_filename): return False - if words not in open(filters_filename, encoding='utf-8').read(): + if not text_in_file(words, filters_filename): return False new_filters_filename = filters_filename + '.new' try: @@ -79,7 +80,7 @@ def remove_global_filter(base_dir: str, words: str) -> bool: filters_filename = base_dir + '/accounts/filters.txt' if not os.path.isfile(filters_filename): return False - if words not in open(filters_filename, encoding='utf-8').read(): + if not text_in_file(words, filters_filename): return False new_filters_filename = filters_filename + '.new' try: diff --git a/follow.py b/follow.py index a9b482f71..0d2f89c07 100644 --- a/follow.py +++ b/follow.py @@ -30,6 +30,7 @@ from utils import get_user_paths from utils import acct_dir from utils import has_group_type from utils import local_actor_url +from utils import text_in_file from acceptreject import create_accept from acceptreject import create_reject from webfinger import webfinger_handle @@ -115,8 +116,7 @@ def _remove_from_follow_base(base_dir: str, ' to remove ' + handle + ' from') return accept_deny_actor = None - if accept_or_deny_handle not in open(approve_follows_filename, - encoding='utf-8').read(): + if not text_in_file(accept_or_deny_handle, approve_follows_filename): # is this stored in the file as an actor rather than a handle? accept_deny_nickname = accept_or_deny_handle.split('@')[0] accept_deny_domain = accept_or_deny_handle.split('@')[1] @@ -317,8 +317,7 @@ def unfollow_account(base_dir: str, nickname: str, domain: str, print('DEBUG: follow file ' + filename + ' was not found') return False handle_to_unfollow_lower = handle_to_unfollow.lower() - if handle_to_unfollow_lower not in open(filename, - encoding='utf-8').read().lower(): + if not text_in_file(handle_to_unfollow_lower, filename, False): if debug: print('DEBUG: handle to unfollow ' + handle_to_unfollow + ' is not in ' + filename) @@ -708,8 +707,7 @@ def store_follow_request(base_dir: str, approve_handle = '!' + approve_handle if os.path.isfile(approve_follows_filename): - if approve_handle not in open(approve_follows_filename, - encoding='utf-8').read(): + if not text_in_file(approve_handle, approve_follows_filename): try: with open(approve_follows_filename, 'a+', encoding='utf-8') as fp_approve: diff --git a/happening.py b/happening.py index 49df1a9d1..c96f16e21 100644 --- a/happening.py +++ b/happening.py @@ -24,6 +24,7 @@ from utils import get_display_name from utils import delete_post from utils import get_status_number from utils import get_full_domain +from utils import text_in_file from filters import is_filtered from context import get_individual_post_context from session import get_method @@ -70,8 +71,7 @@ def _remove_event_from_timeline(event_id: str, tl_events_filename: str) -> None: """Removes the given event Id from the timeline """ - if event_id + '\n' not in open(tl_events_filename, - encoding='utf-8').read(): + if not text_in_file(event_id + '\n', tl_events_filename): return with open(tl_events_filename, 'r', encoding='utf-8') as fp_tl: @@ -760,7 +760,7 @@ def remove_calendar_event(base_dir: str, nickname: str, domain: str, return if '/' in message_id: message_id = message_id.replace('/', '#') - if message_id not in open(calendar_filename, encoding='utf-8').read(): + if not text_in_file(message_id, calendar_filename): return lines = None with open(calendar_filename, 'r', encoding='utf-8') as fp_cal: diff --git a/inbox.py b/inbox.py index 07b53c5f2..a7bc55008 100644 --- a/inbox.py +++ b/inbox.py @@ -18,6 +18,7 @@ from languages import understood_post_language from like import update_likes_collection from reaction import update_reaction_collection from reaction import valid_emoji_content +from utils import text_in_file from utils import get_media_descriptions_from_post from utils import get_summary_from_post from utils import delete_cached_html @@ -2557,8 +2558,7 @@ def populate_replies(base_dir: str, http_prefix: str, domain: str, encoding='utf-8')) if num_lines > max_replies: return False - if message_id not in open(post_replies_filename, - encoding='utf-8').read(): + if not text_in_file(message_id, post_replies_filename): try: with open(post_replies_filename, 'a+', encoding='utf-8') as replies_file: @@ -2875,7 +2875,7 @@ def _like_notify(base_dir: str, domain: str, onion_domain: str, like_file = account_dir + '/.newLike' if os.path.isfile(like_file): - if '##sent##' not in open(like_file).read(): + if not text_in_file('##sent##', like_file): return liker_nickname = get_nickname_from_actor(actor) @@ -2937,7 +2937,7 @@ def _reaction_notify(base_dir: str, domain: str, onion_domain: str, reaction_file = account_dir + '/.newReaction' if os.path.isfile(reaction_file): - if '##sent##' not in open(reaction_file, encoding='utf-8').read(): + if not text_in_file('##sent##', reaction_file): return reaction_nickname = get_nickname_from_actor(actor) @@ -4816,7 +4816,7 @@ def _receive_follow_request(session, session_onion, session_i2p, print('Updating followers file: ' + followers_filename + ' adding ' + approve_handle) if os.path.isfile(followers_filename): - if approve_handle not in open(followers_filename).read(): + if not text_in_file(approve_handle, followers_filename): group_account = \ has_group_type(base_dir, message_json['actor'], person_cache) diff --git a/manualapprove.py b/manualapprove.py index bda5a8f5e..e78df3a7d 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -16,6 +16,7 @@ from utils import remove_domain_port from utils import get_port_from_domain from utils import get_user_paths from utils import acct_dir +from utils import text_in_file from threads import thread_with_trace from session import create_session @@ -115,7 +116,7 @@ def _approve_follower_handle(account_dir: str, approve_handle: str) -> None: """ approved_filename = account_dir + '/approved.txt' if os.path.isfile(approved_filename): - if approve_handle not in open(approved_filename).read(): + if not text_in_file(approve_handle, approved_filename): try: with open(approved_filename, 'a+') as approved_file: approved_file.write(approve_handle + '\n') @@ -280,8 +281,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p, # update the followers print('Manual follow accept: updating ' + followers_filename) if os.path.isfile(followers_filename): - if approve_handle_full not in open(followers_filename, - encoding='utf-8').read(): + if not text_in_file(approve_handle_full, followers_filename): try: with open(followers_filename, 'r+', encoding='utf-8') as followers_file: diff --git a/newsdaemon.py b/newsdaemon.py index 5019803b7..11180de28 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -34,6 +34,7 @@ from utils import get_status_number from utils import clear_from_post_caches from utils import dangerous_markup from utils import local_actor_url +from utils import text_in_file from inbox import store_hash_tags from session import create_session @@ -46,7 +47,7 @@ def _update_feeds_outbox_index(base_dir: str, domain: str, index_filename = base_path + '/outbox.index' if os.path.isfile(index_filename): - if post_id not in open(index_filename, encoding='utf-8').read(): + if not text_in_file(post_id, index_filename): try: with open(index_filename, 'r+') as feeds_file: content = feeds_file.read() diff --git a/notifyOnPost.py b/notifyOnPost.py index f25fa9b48..2a2013a7f 100644 --- a/notifyOnPost.py +++ b/notifyOnPost.py @@ -10,6 +10,7 @@ __module_group__ = "Calendar" import os from utils import remove_domain_port from utils import acct_dir +from utils import text_in_file def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, @@ -30,8 +31,7 @@ def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, handle = following_nickname + '@' + following_domain # check that you are following this handle - if handle + '\n' not in open(following_filename, - encoding='utf-8').read(): + if not text_in_file(handle + '\n', following_filename): print('WARN: ' + handle + ' is not in ' + following_filename) return diff --git a/person.py b/person.py index 13b08117d..25b975aa9 100644 --- a/person.py +++ b/person.py @@ -62,6 +62,7 @@ from utils import get_user_paths from utils import get_group_paths from utils import local_actor_url from utils import dangerous_svg +from utils import text_in_file from session import create_session from session import get_json from webfinger import webfinger_handle @@ -1207,7 +1208,7 @@ def _remove_tags_for_nickname(base_dir: str, nickname: str, continue if not os.path.isfile(tag_filename): continue - if match_str not in open(tag_filename, encoding='utf-8').read(): + if not text_in_file(match_str, tag_filename): continue lines = [] with open(tag_filename, 'r', encoding='utf-8') as fp_tag: @@ -1358,8 +1359,7 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str, snoozed_filename = acct_dir(base_dir, nickname, domain) + '/snoozed.txt' if not os.path.isfile(snoozed_filename): return False - if snooze_actor + ' ' not in open(snoozed_filename, - encoding='utf-8').read(): + if not text_in_file(snooze_actor + ' ', snoozed_filename): return False # remove the snooze entry if it has timed out replace_str = None @@ -1428,8 +1428,7 @@ def person_unsnooze(base_dir: str, nickname: str, domain: str, snoozed_filename = account_dir + '/snoozed.txt' if not os.path.isfile(snoozed_filename): return - if snooze_actor + ' ' not in open(snoozed_filename, - encoding='utf-8').read(): + if not text_in_file(snooze_actor + ' ', snoozed_filename): return replace_str = None with open(snoozed_filename, 'r', encoding='utf-8') as snoozed_file: diff --git a/posts.py b/posts.py index 5bb6577b0..03a5ed389 100644 --- a/posts.py +++ b/posts.py @@ -32,6 +32,7 @@ from webfinger import webfinger_handle from httpsig import create_signed_header from siteactive import site_is_active from languages import understood_post_language +from utils import text_in_file from utils import get_media_descriptions_from_post from utils import valid_hash_tag from utils import get_audio_extensions @@ -968,7 +969,7 @@ def _update_hashtags_index(base_dir: str, tag: {}, new_post_id: str) -> None: tags_filename) else: # prepend to tags index file - if tagline not in open(tags_filename, encoding='utf-8').read(): + if not text_in_file(tagline, tags_filename): try: with open(tags_filename, 'r+', encoding='utf-8') as tags_file: content = tags_file.read() @@ -990,8 +991,7 @@ def _add_schedule_post(base_dir: str, nickname: str, domain: str, index_str = event_date_str + ' ' + post_id.replace('/', '#') if os.path.isfile(schedule_index_filename): - if index_str not in open(schedule_index_filename, - encoding='utf-8').read(): + if not text_in_file(index_str, schedule_index_filename): try: with open(schedule_index_filename, 'r+', encoding='utf-8') as schedule_file: diff --git a/question.py b/question.py index 5ef6829f3..2e36efe36 100644 --- a/question.py +++ b/question.py @@ -12,6 +12,7 @@ from utils import locate_post from utils import load_json from utils import save_json from utils import has_object_dict +from utils import text_in_file def question_update_votes(base_dir: str, nickname: str, domain: str, @@ -74,8 +75,7 @@ def question_update_votes(base_dir: str, nickname: str, domain: str, except OSError: print('EX: unable to write voters file ' + voters_filename) else: - if reply_json['actor'] not in open(voters_filename, - encoding='utf-8').read(): + if not text_in_file(reply_json['actor'], voters_filename): # append to the voters file try: with open(voters_filename, 'a+', diff --git a/roles.py b/roles.py index 7fc649d79..b0ab280d1 100644 --- a/roles.py +++ b/roles.py @@ -13,6 +13,7 @@ from utils import save_json from utils import get_status_number from utils import remove_domain_port from utils import acct_dir +from utils import text_in_file def _clear_role_status(base_dir: str, role: str) -> None: @@ -28,8 +29,7 @@ def _clear_role_status(base_dir: str, role: str) -> None: if not filename.endswith(".json"): continue filename = os.path.join(base_dir + '/accounts/', filename) - if '"' + role + '"' not in open(filename, - encoding='utf-8').read(): + if not text_in_file('"' + role + '"', filename): continue actor_json = load_json(filename) if not actor_json: diff --git a/tests.py b/tests.py index 8168c2c33..996ac489d 100644 --- a/tests.py +++ b/tests.py @@ -54,6 +54,7 @@ from follow import clear_followers from follow import send_follow_request_via_server from follow import send_unfollow_request_via_server from siteactive import site_is_active +from utils import text_in_file from utils import convert_published_to_local_timezone from utils import convert_to_snake_case from utils import get_sha_256 @@ -3467,10 +3468,10 @@ def test_client_to_server(base_dir: str): assert os.path.isfile(bob_followers_filename) assert os.path.isfile(alice_following_filename) - assert 'alice@' + alice_domain + ':' + str(alice_port) \ - not in open(bob_followers_filename, encoding='utf-8').read() - assert 'bob@' + bob_domain + ':' + str(bob_port) \ - not in open(alice_following_filename, encoding='utf-8').read() + test_str = 'alice@' + alice_domain + ':' + str(alice_port) + assert not text_in_file(test_str, bob_followers_filename) + test_str = 'bob@' + bob_domain + ':' + str(bob_port) + assert not text_in_file(test_str, alice_following_filename) assert valid_inbox(bob_dir, 'bob', bob_domain) assert valid_inbox_filenames(bob_dir, 'bob', bob_domain, alice_domain, alice_port) diff --git a/utils.py b/utils.py index 303e6d5c4..d4ab98c05 100644 --- a/utils.py +++ b/utils.py @@ -40,13 +40,17 @@ INVALID_CHARACTERS = ( ) -def text_in_file(text: str, filename: str) -> bool: +def text_in_file(text: str, filename: str, case_sensitive: bool = True) -> bool: """is the given text in the given file? """ + if not case_sensitive: + text = text.lower() try: with open(filename, 'r', encoding='utf-8') as file: content = file.read() if content: + if not case_sensitive: + content = content.lower() if text in content: return True except OSError: