diff --git a/acceptreject.py b/acceptreject.py index 2b7a7021f..a58578611 100644 --- a/acceptreject.py +++ b/acceptreject.py @@ -75,7 +75,7 @@ def create_reject(base_dir: str, federation_list: [], http_prefix, object_json, 'Reject') -def _accept_follow(base_dir: str, domain: str, message_json: {}, +def _accept_follow(base_dir: str, message_json: {}, federation_list: [], debug: bool, curr_domain: str, onion_domain: str, i2p_domain: str) -> None: @@ -165,7 +165,7 @@ def _accept_follow(base_dir: str, domain: str, message_json: {}, acct_dir(base_dir, nickname, accepted_domain_full) + '/unfollowed.txt' if os.path.isfile(unfollowed_filename): if followed_nickname + '@' + followed_domain_full in \ - open(unfollowed_filename).read(): + open(unfollowed_filename, encoding='utf-8').read(): if debug: print('DEBUG: follow accept arrived for ' + nickname + '@' + accepted_domain_full + @@ -195,14 +195,8 @@ def _accept_follow(base_dir: str, domain: str, message_json: {}, followed_nickname + '@' + followed_domain) -def receive_accept_reject(session, base_dir: str, - http_prefix: str, domain: str, port: int, - send_threads: [], post_log: [], - cached_webfingers: {}, - person_cache: {}, message_json: {}, - federation_list: [], - debug: bool, - curr_domain: str, +def receive_accept_reject(base_dir: str, domain: str, message_json: {}, + federation_list: [], debug: bool, curr_domain: str, onion_domain: str, i2p_domain: str) -> bool: """Receives an Accept or Reject within the POST section of HTTPServer """ @@ -229,7 +223,7 @@ def receive_accept_reject(session, base_dir: str, ' does not contain a nickname. ' + 'Assuming single user instance.') # receive follow accept - _accept_follow(base_dir, domain, message_json, federation_list, debug, + _accept_follow(base_dir, message_json, federation_list, debug, curr_domain, onion_domain, i2p_domain) if debug: print('DEBUG: Uh, ' + message_json['type'] + ', I guess') diff --git a/announce.py b/announce.py index ef1dacac9..1bc3c538f 100644 --- a/announce.py +++ b/announce.py @@ -331,8 +331,7 @@ def send_announce_via_server(base_dir: str, session, def send_undo_announce_via_server(base_dir: str, session, undo_post_json_object: {}, nickname: str, password: str, - domain: str, port: int, - http_prefix: str, repeat_object_url: str, + domain: str, port: int, http_prefix: str, cached_webfingers: {}, person_cache: {}, debug: bool, project_version: str, signing_priv_key_pem: str) -> {}: @@ -413,8 +412,7 @@ def send_undo_announce_via_server(base_dir: str, session, def outbox_undo_announce(recent_posts_cache: {}, - base_dir: str, http_prefix: str, - nickname: str, domain: str, port: int, + base_dir: str, nickname: str, domain: str, message_json: {}, debug: bool) -> None: """ When an undo announce is received by the outbox from c2s """ diff --git a/auth.py b/auth.py index e137eeab6..e47dd6cbb 100644 --- a/auth.py +++ b/auth.py @@ -143,7 +143,7 @@ def authorize_basic(base_dir: str, path: str, auth_header: str, return False provided_password = plain.split(':')[1] try: - with open(password_file, 'r') as passfile: + with open(password_file, 'r', encoding='utf-8') as passfile: for line in passfile: if not line.startswith(nickname + ':'): continue @@ -177,10 +177,11 @@ def store_basic_credentials(base_dir: str, password_file = base_dir + '/accounts/passwords' store_str = nickname + ':' + _hash_password(password) if os.path.isfile(password_file): - if nickname + ':' in open(password_file).read(): + if nickname + ':' in open(password_file, encoding='utf-8').read(): try: - with open(password_file, 'r') as fin: - with open(password_file + '.new', 'w+') as fout: + with open(password_file, 'r', encoding='utf-8') as fin: + with open(password_file + '.new', 'w+', + encoding='utf-8') as fout: for line in fin: if not line.startswith(nickname + ':'): fout.write(line) @@ -199,14 +200,14 @@ def store_basic_credentials(base_dir: str, else: # append to password file try: - with open(password_file, 'a+') as passfile: + with open(password_file, 'a+', encoding='utf-8') as passfile: passfile.write(store_str + '\n') except OSError: print('EX: unable to append password') return False else: try: - with open(password_file, 'w+') as passfile: + with open(password_file, 'w+', encoding='utf-8') as passfile: passfile.write(store_str + '\n') except OSError: print('EX: unable to create password file') @@ -221,8 +222,9 @@ def remove_password(base_dir: str, nickname: str) -> None: password_file = base_dir + '/accounts/passwords' if os.path.isfile(password_file): try: - with open(password_file, 'r') as fin: - with open(password_file + '.new', 'w+') as fout: + with open(password_file, 'r', encoding='utf-8') as fin: + with open(password_file + '.new', 'w+', + encoding='utf-8') as fout: for line in fin: if not line.startswith(nickname + ':'): fout.write(line) @@ -289,7 +291,7 @@ def record_login_failure(base_dir: str, ip_address: str, curr_time = datetime.datetime.utcnow() curr_time_str = curr_time.strftime("%Y-%m-%d %H:%M:%SZ") try: - with open(failure_log, write_type) as fp_fail: + with open(failure_log, write_type, encoding='utf-8') as fp_fail: # here we use a similar format to an ssh log, so that # systems such as fail2ban can parse it fp_fail.write(curr_time_str + ' ' + diff --git a/blocking.py b/blocking.py index f73bc93d1..9d5c506c9 100644 --- a/blocking.py +++ b/blocking.py @@ -46,11 +46,12 @@ def add_global_block(base_dir: str, # is the handle already blocked? block_handle = block_nickname + '@' + block_domain if os.path.isfile(blocking_filename): - if block_handle in open(blocking_filename).read(): + if block_handle in open(blocking_filename, + encoding='utf-8').read(): return False # block an account handle or domain try: - with open(blocking_filename, 'a+') as block_file: + with open(blocking_filename, 'a+', encoding='utf-8') as block_file: block_file.write(block_handle + '\n') except OSError: print('EX: unable to save blocked handle ' + block_handle) @@ -59,11 +60,12 @@ def add_global_block(base_dir: str, block_hashtag = block_nickname # is the hashtag already blocked? if os.path.isfile(blocking_filename): - if block_hashtag + '\n' in open(blocking_filename).read(): + if block_hashtag + '\n' in \ + open(blocking_filename, encoding='utf-8').read(): return False # block a hashtag try: - with open(blocking_filename, 'a+') as block_file: + with open(blocking_filename, 'a+', encoding='utf-8') as block_file: block_file.write(block_hashtag + '\n') except OSError: print('EX: unable to save blocked hashtag ' + block_hashtag) @@ -83,17 +85,20 @@ def add_block(base_dir: str, nickname: str, domain: str, blocking_filename = acct_dir(base_dir, nickname, domain) + '/blocking.txt' block_handle = block_nickname + '@' + block_domain if os.path.isfile(blocking_filename): - if block_handle + '\n' in open(blocking_filename).read(): + if block_handle + '\n' in open(blocking_filename, + encoding='utf-8').read(): return False # if we are following then unfollow following_filename = \ acct_dir(base_dir, nickname, domain) + '/following.txt' if os.path.isfile(following_filename): - if block_handle + '\n' in open(following_filename).read(): + if block_handle + '\n' in open(following_filename, + encoding='utf-8').read(): following_str = '' try: - with open(following_filename, 'r') as foll_file: + with open(following_filename, 'r', + encoding='utf-8') as foll_file: following_str = foll_file.read() except OSError: print('EX: Unable to read following ' + following_filename) @@ -103,7 +108,8 @@ def add_block(base_dir: str, nickname: str, domain: str, following_str = following_str.replace(block_handle + '\n', '') try: - with open(following_filename, 'w+') as foll_file: + with open(following_filename, 'w+', + encoding='utf-8') as foll_file: foll_file.write(following_str) except OSError: print('EX: Unable to write following ' + following_str) @@ -113,10 +119,12 @@ def add_block(base_dir: str, nickname: str, domain: str, followers_filename = \ acct_dir(base_dir, nickname, domain) + '/followers.txt' if os.path.isfile(followers_filename): - if block_handle + '\n' in open(followers_filename).read(): + if block_handle + '\n' in open(followers_filename, + encoding='utf-8').read(): followers_str = '' try: - with open(followers_filename, 'r') as foll_file: + with open(followers_filename, 'r', + encoding='utf-8') as foll_file: followers_str = foll_file.read() except OSError: print('EX: Unable to read followers ' + followers_filename) @@ -126,14 +134,15 @@ def add_block(base_dir: str, nickname: str, domain: str, followers_str = followers_str.replace(block_handle + '\n', '') try: - with open(followers_filename, 'w+') as foll_file: + with open(followers_filename, 'w+', + encoding='utf-8') as foll_file: foll_file.write(followers_str) except OSError: print('EX: Unable to write followers ' + followers_str) return False try: - with open(blocking_filename, 'a+') as block_file: + with open(blocking_filename, 'a+', encoding='utf-8') as block_file: block_file.write(block_handle + '\n') except OSError: print('EX: unable to append block handle ' + block_handle) @@ -150,10 +159,13 @@ def remove_global_block(base_dir: str, if not unblock_nickname.startswith('#'): unblock_handle = unblock_nickname + '@' + unblock_domain if os.path.isfile(unblocking_filename): - if unblock_handle in open(unblocking_filename).read(): + if unblock_handle in open(unblocking_filename, + encoding='utf-8').read(): try: - with open(unblocking_filename, 'r') as fp_unblock: - with open(unblocking_filename + '.new', 'w+') as fpnew: + with open(unblocking_filename, 'r', + encoding='utf-8') as fp_unblock: + with open(unblocking_filename + '.new', 'w+', + encoding='utf-8') as fpnew: for line in fp_unblock: handle = \ line.replace('\n', '').replace('\r', '') @@ -175,10 +187,13 @@ def remove_global_block(base_dir: str, else: unblock_hashtag = unblock_nickname if os.path.isfile(unblocking_filename): - if unblock_hashtag + '\n' in open(unblocking_filename).read(): + if unblock_hashtag + '\n' in open(unblocking_filename, + encoding='utf-8').read(): try: - with open(unblocking_filename, 'r') as fp_unblock: - with open(unblocking_filename + '.new', 'w+') as fpnew: + with open(unblocking_filename, 'r', + encoding='utf-8') as fp_unblock: + with open(unblocking_filename + '.new', 'w+', + encoding='utf-8') as fpnew: for line in fp_unblock: block_line = \ line.replace('\n', '').replace('\r', '') @@ -209,10 +224,13 @@ def remove_block(base_dir: str, nickname: str, domain: str, acct_dir(base_dir, nickname, domain) + '/blocking.txt' unblock_handle = unblock_nickname + '@' + unblock_domain if os.path.isfile(unblocking_filename): - if unblock_handle in open(unblocking_filename).read(): + if unblock_handle in open(unblocking_filename, + encoding='utf-8').read(): try: - with open(unblocking_filename, 'r') as fp_unblock: - with open(unblocking_filename + '.new', 'w+') as fpnew: + with open(unblocking_filename, 'r', + encoding='utf-8') as fp_unblock: + with open(unblocking_filename + '.new', 'w+', + encoding='utf-8') as fpnew: for line in fp_unblock: handle = line.replace('\n', '').replace('\r', '') if unblock_handle not in line: @@ -244,7 +262,8 @@ def is_blocked_hashtag(base_dir: str, hashtag: str) -> bool: hashtag = hashtag.strip('\n').strip('\r') if not hashtag.startswith('#'): hashtag = '#' + hashtag - if hashtag + '\n' in open(global_blocking_filename).read(): + if hashtag + '\n' in open(global_blocking_filename, + encoding='utf-8').read(): return True return False @@ -263,7 +282,8 @@ def get_domain_blocklist(base_dir: str) -> str: if not os.path.isfile(global_blocking_filename): return blocked_str try: - with open(global_blocking_filename, 'r') as fp_blocked: + with open(global_blocking_filename, 'r', + encoding='utf-8') as fp_blocked: blocked_str += fp_blocked.read() except OSError: print('EX: unable to read ' + global_blocking_filename) @@ -287,7 +307,8 @@ def update_blocked_cache(base_dir: str, if not os.path.isfile(global_blocking_filename): return blocked_cache_last_updated try: - with open(global_blocking_filename, 'r') as fp_blocked: + with open(global_blocking_filename, 'r', + encoding='utf-8') as fp_blocked: blocked_lines = fp_blocked.readlines() # remove newlines for index, _ in enumerate(blocked_lines): @@ -337,7 +358,8 @@ def is_blocked_domain(base_dir: str, domain: str, global_blocking_filename = base_dir + '/accounts/blocking.txt' if os.path.isfile(global_blocking_filename): try: - with open(global_blocking_filename, 'r') as fp_blocked: + with open(global_blocking_filename, 'r', + encoding='utf-8') as fp_blocked: blocked_str = fp_blocked.read() if '*@' + domain + '\n' in blocked_str: return True @@ -351,10 +373,11 @@ def is_blocked_domain(base_dir: str, domain: str, allow_filename = base_dir + '/accounts/allowedinstances.txt' # instance allow list if not short_domain: - if domain not in open(allow_filename).read(): + if domain not in open(allow_filename, encoding='utf-8').read(): return True else: - if short_domain not in open(allow_filename).read(): + if short_domain not in open(allow_filename, + encoding='utf-8').read(): return True return False @@ -384,43 +407,49 @@ def is_blocked(base_dir: str, nickname: str, domain: str, else: global_blocks_filename = base_dir + '/accounts/blocking.txt' if os.path.isfile(global_blocks_filename): - if '*@' + block_domain in open(global_blocks_filename).read(): + if '*@' + block_domain in open(global_blocks_filename, + encoding='utf-8').read(): return True if block_handle: block_str = block_handle + '\n' - if block_str in open(global_blocks_filename).read(): + if block_str in open(global_blocks_filename, + encoding='utf-8').read(): return True else: # instance allow list allow_filename = base_dir + '/accounts/allowedinstances.txt' short_domain = _get_short_domain(block_domain) if not short_domain: - if block_domain + '\n' not in open(allow_filename).read(): + if block_domain + '\n' not in open(allow_filename, + encoding='utf-8').read(): return True else: - if short_domain + '\n' not in open(allow_filename).read(): + if short_domain + '\n' not in open(allow_filename, + encoding='utf-8').read(): return True # account level allow list account_dir = acct_dir(base_dir, nickname, domain) allow_filename = account_dir + '/allowedinstances.txt' if os.path.isfile(allow_filename): - if block_domain + '\n' not in open(allow_filename).read(): + if block_domain + '\n' not in open(allow_filename, + encoding='utf-8').read(): return True # account level block list blocking_filename = account_dir + '/blocking.txt' if os.path.isfile(blocking_filename): - if '*@' + block_domain + '\n' in open(blocking_filename).read(): + if '*@' + block_domain + '\n' in open(blocking_filename, + encoding='utf-8').read(): return True if block_handle: - if block_handle + '\n' in open(blocking_filename).read(): + if block_handle + '\n' in open(blocking_filename, + encoding='utf-8').read(): return True return False -def outbox_block(base_dir: str, http_prefix: str, - nickname: str, domain: str, port: int, +def outbox_block(base_dir: str, nickname: str, domain: str, message_json: {}, debug: bool) -> bool: """ When a block request is received by the outbox from c2s """ @@ -606,7 +635,8 @@ def mute_post(base_dir: str, nickname: str, domain: str, port: int, print('MUTE: cached post not found ' + cached_post_filename) try: - with open(post_filename + '.muted', 'w+') as mute_file: + with open(post_filename + '.muted', 'w+', + encoding='utf-8') as mute_file: mute_file.write('\n') except OSError: print('EX: Failed to save mute file ' + post_filename + '.muted') @@ -926,7 +956,8 @@ def set_broch_mode(base_dir: str, domain_full: str, enabled: bool) -> None: if not os.path.isfile(following_filename): continue try: - with open(following_filename, 'r') as foll_file: + with open(following_filename, 'r', + encoding='utf-8') as foll_file: follow_list = foll_file.readlines() for handle in follow_list: if '@' not in handle: @@ -942,7 +973,8 @@ def set_broch_mode(base_dir: str, domain_full: str, enabled: bool) -> None: # write the allow file try: - with open(allow_filename, 'w+') as allow_file: + with open(allow_filename, 'w+', + encoding='utf-8') as allow_file: allow_file.write(domain_full + '\n') for allowed in allowed_domains: allow_file.write(allowed + '\n') @@ -1083,7 +1115,7 @@ def add_cw_from_lists(post_json_object: {}, cw_lists: {}, translate: {}, post_json_object['object']['sensitive'] = True -def get_cw_list_variable(listName: str) -> str: +def get_cw_list_variable(list_name: str) -> str: """Returns the variable associated with a CW list """ - return 'list' + listName.replace(' ', '').replace("'", '') + return 'list' + list_name.replace(' ', '').replace("'", '') diff --git a/blog.py b/blog.py index 130dd8880..d60e73b69 100644 --- a/blog.py +++ b/blog.py @@ -41,7 +41,7 @@ from cache import get_person_from_cache def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, nickname: str, domain: str, domain_full: str, - post_id: str, depth=0) -> int: + post_id: str, depth: int = 0) -> int: """Returns the number of replies on the post This is recursive, so can handle replies to replies """ @@ -73,7 +73,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, replies = 0 lines = [] try: - with open(post_filename, 'r') as post_file: + with open(post_filename, 'r', encoding='utf-8') as post_file: lines = post_file.readlines() except OSError: print('EX: failed to read blog ' + post_filename) @@ -96,7 +96,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, print('Rewriting ' + post_filename + ' to remove ' + str(len(removals)) + ' entries') try: - with open(post_filename, 'w+') as post_file: + with open(post_filename, 'w+', encoding='utf-8') as post_file: for reply_post_id in lines: reply_post_id = \ reply_post_id.replace('\n', '').replace('\r', '') @@ -111,7 +111,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, def _get_blog_replies(base_dir: str, http_prefix: str, translate: {}, nickname: str, domain: str, domain_full: str, - post_id: str, depth=0) -> str: + post_id: str, depth: int = 0) -> str: """Returns a string containing html blog posts """ if depth > 4: @@ -140,7 +140,8 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {}, post_id.replace('/', '#') + '.html' if os.path.isfile(post_filename): try: - with open(post_filename, 'r') as post_file: + with open(post_filename, 'r', + encoding='utf-8') as post_file: return post_file.read() + '\n' except OSError: print('EX: unable to read blog 3 ' + post_filename) @@ -148,7 +149,7 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {}, lines = [] try: - with open(post_filename, 'r') as post_file: + with open(post_filename, 'r', encoding='utf-8') as post_file: lines = post_file.readlines() except OSError: print('EX: unable to read blog 4 ' + post_filename) @@ -165,7 +166,7 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {}, if not os.path.isfile(post_filename): continue try: - with open(post_filename, 'r') as post_file: + with open(post_filename, 'r', encoding='utf-8') as post_file: replies_str += post_file.read() + '\n' except OSError: print('EX: unable to read blog replies ' + post_filename) @@ -271,7 +272,7 @@ def _html_blog_post_content(debug: bool, session, authorized: bool, person_url = local_actor_url(http_prefix, nickname, domain_full) actor_json = \ - get_person_from_cache(base_dir, person_url, person_cache, False) + get_person_from_cache(base_dir, person_url, person_cache) languages_understood = [] if actor_json: languages_understood = get_actor_languages_list(actor_json) @@ -354,11 +355,8 @@ def _html_blog_post_content(debug: bool, session, authorized: bool, return blog_str -def _html_blog_post_rss2(authorized: bool, - base_dir: str, http_prefix: str, translate: {}, - nickname: str, domain: str, domain_full: str, - post_json_object: {}, - handle: str, restrict_to_domain: bool, +def _html_blog_post_rss2(domain: str, post_json_object: {}, + restrict_to_domain: bool, system_language: str) -> str: """Returns the RSS version 2 feed for a single blog post """ @@ -389,11 +387,8 @@ def _html_blog_post_rss2(authorized: bool, return rss_str -def _html_blog_post_rss3(authorized: bool, - base_dir: str, http_prefix: str, translate: {}, - nickname: str, domain: str, domain_full: str, - post_json_object: {}, - handle: str, restrict_to_domain: bool, +def _html_blog_post_rss3(domain: str, post_json_object: {}, + restrict_to_domain: bool, system_language: str) -> str: """Returns the RSS version 3 feed for a single blog post """ @@ -605,8 +600,7 @@ def html_blog_page(authorized: bool, session, return blog_str + html_footer() -def html_blog_page_rss2(authorized: bool, session, - base_dir: str, http_prefix: str, translate: {}, +def html_blog_page_rss2(base_dir: str, http_prefix: str, translate: {}, nickname: str, domain: str, port: int, no_of_items: int, page_number: int, include_header: bool, system_language: str) -> str: @@ -646,19 +640,14 @@ def html_blog_page_rss2(authorized: bool, session, continue blog_rss2 += \ - _html_blog_post_rss2(authorized, base_dir, - http_prefix, translate, - nickname, domain, - domain_full, item, - None, True, system_language) + _html_blog_post_rss2(domain, item, True, system_language) if include_header: return blog_rss2 + rss2footer() return blog_rss2 -def html_blog_page_rss3(authorized: bool, session, - base_dir: str, http_prefix: str, translate: {}, +def html_blog_page_rss3(base_dir: str, http_prefix: str, nickname: str, domain: str, port: int, no_of_items: int, page_number: int, system_language: str) -> str: @@ -668,8 +657,6 @@ def html_blog_page_rss3(authorized: bool, session, '\n' in nickname or '\r' in nickname: return None - domain_full = get_full_domain(domain, port) - blog_rss3 = '' blogs_index = acct_dir(base_dir, nickname, domain) + '/tlblogs.index' @@ -690,12 +677,7 @@ def html_blog_page_rss3(authorized: bool, session, continue blog_rss3 += \ - _html_blog_post_rss3(authorized, base_dir, - http_prefix, translate, - nickname, domain, - domain_full, item, - None, True, - system_language) + _html_blog_post_rss3(domain, item, True, system_language) return blog_rss3 @@ -760,7 +742,7 @@ def html_blog_view(authorized: bool, domain_full = get_full_domain(domain, port) - for subdir, dirs, files in os.walk(base_dir + '/accounts'): + for _, dirs, _ in os.walk(base_dir + '/accounts'): for acct in dirs: if not is_account_dir(acct): continue @@ -778,9 +760,7 @@ def html_blog_view(authorized: bool, def html_edit_blog(media_instance: bool, translate: {}, - base_dir: str, http_prefix: str, - path: str, - page_number: int, + base_dir: str, path: str, page_number: int, nickname: str, domain: str, post_url: str, system_language: str) -> str: """Edit a blog post after it was created @@ -800,7 +780,8 @@ def html_edit_blog(media_instance: bool, translate: {}, if os.path.isfile(base_dir + '/accounts/newpost.txt'): try: - with open(base_dir + '/accounts/newpost.txt', 'r') as file: + with open(base_dir + '/accounts/newpost.txt', 'r', + encoding='utf-8') as file: edit_blog_text = '

' + file.read() + '

' except OSError: print('EX: unable to read ' + base_dir + '/accounts/newpost.txt') @@ -948,7 +929,8 @@ def path_contains_blog_link(base_dir: str, acct_dir(base_dir, nickname, domain) + '/tlblogs.index' if not os.path.isfile(blog_index_filename): return None, None - if '#' + user_ending2[1] + '.' not in open(blog_index_filename).read(): + if '#' + user_ending2[1] + '.' not in open(blog_index_filename, + encoding='utf-8').read(): return None, None message_id = local_actor_url(http_prefix, nickname, domain_full) + \ '/statuses/' + user_ending2[1] diff --git a/bookmarks.py b/bookmarks.py index 6d8c78d1e..307b544f6 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -34,7 +34,6 @@ from session import post_json def undo_bookmarks_collection_entry(recent_posts_cache: {}, base_dir: str, post_filename: str, - object_url: str, actor: str, domain: str, debug: bool) -> None: """Undoes a bookmark for a particular actor @@ -72,17 +71,20 @@ def undo_bookmarks_collection_entry(recent_posts_cache: {}, else: bookmark_index = post_filename.strip() bookmark_index = bookmark_index.replace('\n', '').replace('\r', '') - if bookmark_index not in open(bookmarks_index_filename).read(): + if bookmark_index not in open(bookmarks_index_filename, + encoding='utf-8').read(): return index_str = '' try: - with open(bookmarks_index_filename, 'r') as index_file: + with open(bookmarks_index_filename, 'r', + encoding='utf-8') as index_file: index_str = index_file.read().replace(bookmark_index + '\n', '') except OSError: print('EX: unable to read ' + bookmarks_index_filename) if index_str: try: - with open(bookmarks_index_filename, 'w+') as bmi_file: + with open(bookmarks_index_filename, 'w+', + encoding='utf-8') as bmi_file: bmi_file.write(index_str) except OSError: print('EX: unable to write bookmarks index ' + @@ -164,106 +166,108 @@ def update_bookmarks_collection(recent_posts_cache: {}, """Updates the bookmarks collection within a post """ post_json_object = load_json(post_filename) - if post_json_object: - # remove any cached version of this post so that the - # bookmark icon is changed - nickname = get_nickname_from_actor(actor) - if not nickname: - return - cached_post_filename = \ - get_cached_post_filename(base_dir, nickname, - domain, post_json_object) - if cached_post_filename: - if os.path.isfile(cached_post_filename): - try: - os.remove(cached_post_filename) - except OSError: - if debug: - print('EX: update_bookmarks_collection ' + - 'unable to delete cached post ' + - str(cached_post_filename)) - remove_post_from_cache(post_json_object, recent_posts_cache) + if not post_json_object: + return - if not post_json_object.get('object'): - if debug: - print('DEBUG: no object in bookmarked post ' + - str(post_json_object)) - return - if not object_url.endswith('/bookmarks'): - object_url = object_url + '/bookmarks' - # does this post have bookmarks on it from differenent actors? - if not post_json_object['object'].get('bookmarks'): - if debug: - print('DEBUG: Adding initial bookmarks to ' + object_url) - bookmarks_json = { - "@context": "https://www.w3.org/ns/activitystreams", - 'id': object_url, - 'type': 'Collection', - "totalItems": 1, - 'items': [{ - 'type': 'Bookmark', - 'actor': actor - }] - } - post_json_object['object']['bookmarks'] = bookmarks_json - else: - if not post_json_object['object']['bookmarks'].get('items'): - post_json_object['object']['bookmarks']['items'] = [] - bm_items = post_json_object['object']['bookmarks']['items'] - for bookmark_item in bm_items: - if bookmark_item.get('actor'): - if bookmark_item['actor'] == actor: - return - new_bookmark = { + # remove any cached version of this post so that the + # bookmark icon is changed + nickname = get_nickname_from_actor(actor) + if not nickname: + return + cached_post_filename = \ + get_cached_post_filename(base_dir, nickname, + domain, post_json_object) + if cached_post_filename: + if os.path.isfile(cached_post_filename): + try: + os.remove(cached_post_filename) + except OSError: + if debug: + print('EX: update_bookmarks_collection ' + + 'unable to delete cached post ' + + str(cached_post_filename)) + remove_post_from_cache(post_json_object, recent_posts_cache) + + if not post_json_object.get('object'): + if debug: + print('DEBUG: no object in bookmarked post ' + + str(post_json_object)) + return + if not object_url.endswith('/bookmarks'): + object_url = object_url + '/bookmarks' + # does this post have bookmarks on it from differenent actors? + if not post_json_object['object'].get('bookmarks'): + if debug: + print('DEBUG: Adding initial bookmarks to ' + object_url) + bookmarks_json = { + "@context": "https://www.w3.org/ns/activitystreams", + 'id': object_url, + 'type': 'Collection', + "totalItems": 1, + 'items': [{ 'type': 'Bookmark', 'actor': actor - } - nbook = new_bookmark - bm_it = len(post_json_object['object']['bookmarks']['items']) - post_json_object['object']['bookmarks']['items'].append(nbook) - post_json_object['object']['bookmarks']['totalItems'] = bm_it + }] + } + post_json_object['object']['bookmarks'] = bookmarks_json + else: + if not post_json_object['object']['bookmarks'].get('items'): + post_json_object['object']['bookmarks']['items'] = [] + bm_items = post_json_object['object']['bookmarks']['items'] + for bookmark_item in bm_items: + if bookmark_item.get('actor'): + if bookmark_item['actor'] == actor: + return + new_bookmark = { + 'type': 'Bookmark', + 'actor': actor + } + nbook = new_bookmark + bm_it = len(post_json_object['object']['bookmarks']['items']) + post_json_object['object']['bookmarks']['items'].append(nbook) + post_json_object['object']['bookmarks']['totalItems'] = bm_it - if debug: - print('DEBUG: saving post with bookmarks added') - pprint(post_json_object) + if debug: + print('DEBUG: saving post with bookmarks added') + pprint(post_json_object) - save_json(post_json_object, post_filename) + save_json(post_json_object, post_filename) - # prepend to the index - bookmarks_index_filename = \ - acct_dir(base_dir, nickname, domain) + '/bookmarks.index' - bookmark_index = post_filename.split('/')[-1] - if os.path.isfile(bookmarks_index_filename): - if bookmark_index not in open(bookmarks_index_filename).read(): - try: - with open(bookmarks_index_filename, 'r+') as bmi_file: - content = bmi_file.read() - if bookmark_index + '\n' not in content: - bmi_file.seek(0, 0) - bmi_file.write(bookmark_index + '\n' + content) - if debug: - print('DEBUG: bookmark added to index') - except OSError as ex: - print('WARN: Failed to write entry to bookmarks index ' + - bookmarks_index_filename + ' ' + str(ex)) - else: + # prepend to the index + bookmarks_index_filename = \ + acct_dir(base_dir, nickname, domain) + '/bookmarks.index' + bookmark_index = post_filename.split('/')[-1] + if os.path.isfile(bookmarks_index_filename): + if bookmark_index not in open(bookmarks_index_filename, + encoding='utf-8').read(): try: - with open(bookmarks_index_filename, 'w+') as bm_file: - bm_file.write(bookmark_index + '\n') - except OSError: - print('EX: unable to write bookmarks index ' + - bookmarks_index_filename) + with open(bookmarks_index_filename, 'r+', + encoding='utf-8') as bmi_file: + content = bmi_file.read() + if bookmark_index + '\n' not in content: + bmi_file.seek(0, 0) + bmi_file.write(bookmark_index + '\n' + content) + if debug: + print('DEBUG: bookmark added to index') + except OSError as ex: + print('WARN: Failed to write entry to bookmarks index ' + + bookmarks_index_filename + ' ' + str(ex)) + else: + try: + with open(bookmarks_index_filename, 'w+', + encoding='utf-8') as bm_file: + bm_file.write(bookmark_index + '\n') + except OSError: + print('EX: unable to write bookmarks index ' + + bookmarks_index_filename) def bookmark_post(recent_posts_cache: {}, - session, base_dir: str, federation_list: [], + base_dir: str, federation_list: [], nickname: str, domain: str, port: int, - ccList: [], http_prefix: str, - object_url: str, actorBookmarked: str, - client_to_server: bool, - send_threads: [], post_log: [], - person_cache: {}, cached_webfingers: {}, - debug: bool, project_version: str) -> {}: + cc_list: [], http_prefix: str, + object_url: str, actor_bookmarked: str, + debug: bool) -> {}: """Creates a bookmark actor is the person doing the bookmarking 'to' might be a specific person (actor) whose post was bookmarked @@ -280,14 +284,14 @@ def bookmark_post(recent_posts_cache: {}, 'actor': local_actor_url(http_prefix, nickname, full_domain), 'object': object_url } - if ccList: - if len(ccList) > 0: - new_bookmark_json['cc'] = ccList + if cc_list: + if len(cc_list) > 0: + new_bookmark_json['cc'] = cc_list # Extract the domain and nickname from a statuses link bookmarked_post_nickname = None - if actorBookmarked: - ac_bm = actorBookmarked + if actor_bookmarked: + ac_bm = actor_bookmarked bookmarked_post_nickname = get_nickname_from_actor(ac_bm) _, _ = get_domain_from_actor(ac_bm) else: @@ -313,14 +317,11 @@ def bookmark_post(recent_posts_cache: {}, def undo_bookmark_post(recent_posts_cache: {}, - session, base_dir: str, federation_list: [], + base_dir: str, federation_list: [], nickname: str, domain: str, port: int, - ccList: [], http_prefix: str, - object_url: str, actorBookmarked: str, - client_to_server: bool, - send_threads: [], post_log: [], - person_cache: {}, cached_webfingers: {}, - debug: bool, project_version: str) -> {}: + cc_list: [], http_prefix: str, + object_url: str, actor_bookmarked: str, + debug: bool) -> {}: """Removes a bookmark actor is the person doing the bookmarking 'to' might be a specific person (actor) whose post was bookmarked @@ -341,15 +342,15 @@ def undo_bookmark_post(recent_posts_cache: {}, 'object': object_url } } - if ccList: - if len(ccList) > 0: - new_undo_bookmark_json['cc'] = ccList - new_undo_bookmark_json['object']['cc'] = ccList + if cc_list: + if len(cc_list) > 0: + new_undo_bookmark_json['cc'] = cc_list + new_undo_bookmark_json['object']['cc'] = cc_list # Extract the domain and nickname from a statuses link bookmarked_post_nickname = None - if actorBookmarked: - ac_bm = actorBookmarked + if actor_bookmarked: + ac_bm = actor_bookmarked bookmarked_post_nickname = get_nickname_from_actor(ac_bm) _, _ = get_domain_from_actor(ac_bm) else: @@ -364,7 +365,7 @@ def undo_bookmark_post(recent_posts_cache: {}, return None undo_bookmarks_collection_entry(recent_posts_cache, - base_dir, post_filename, object_url, + base_dir, post_filename, new_undo_bookmark_json['actor'], domain, debug) else: @@ -578,9 +579,10 @@ def outbox_bookmark(recent_posts_cache: {}, print('DEBUG: bookmark Add target is not string') return domain_full = get_full_domain(domain, port) - if not message_json['target'].endswith('://' + domain_full + - '/users/' + nickname + - '/tlbookmarks'): + expected_target = \ + http_prefix + '://' + domain_full + \ + '/users/' + nickname + '/tlbookmarks' + if message_json['target'] != expected_target: if debug: print('DEBUG: bookmark Add target invalid ' + message_json['target']) @@ -634,9 +636,10 @@ def outbox_undo_bookmark(recent_posts_cache: {}, print('DEBUG: unbookmark Remove target is not string') return domain_full = get_full_domain(domain, port) - if not message_json['target'].endswith('://' + domain_full + - '/users/' + nickname + - '/tlbookmarks'): + expected_target = \ + http_prefix + '://' + domain_full + \ + '/users/' + nickname + '/tlbookmarks' + if message_json['target'] != expected_target: if debug: print('DEBUG: unbookmark Remove target invalid ' + message_json['target']) diff --git a/cache.py b/cache.py index 4997946e7..e2bbe3c24 100644 --- a/cache.py +++ b/cache.py @@ -77,8 +77,8 @@ def store_person_in_cache(base_dir: str, person_url: str, save_json(person_json, cache_filename) -def get_person_from_cache(base_dir: str, person_url: str, person_cache: {}, - allow_write_to_file: bool) -> {}: +def get_person_from_cache(base_dir: str, person_url: str, + person_cache: {}) -> {}: """Get an actor from the cache """ # if the actor is not in memory then try to load it from file @@ -157,7 +157,7 @@ def get_person_pub_key(base_dir: str, session, person_url: str, person_url.replace(possible_users_path + 'inbox', '/inbox') break person_json = \ - get_person_from_cache(base_dir, person_url, person_cache, True) + get_person_from_cache(base_dir, person_url, person_cache) if not person_json: if debug: print('DEBUG: Obtaining public key for ' + person_url) diff --git a/categories.py b/categories.py index 3a0838a61..1ee488e63 100644 --- a/categories.py +++ b/categories.py @@ -29,7 +29,7 @@ def get_hashtag_category(base_dir: str, hashtag: str) -> str: category_str = None try: - with open(category_filename, 'r') as category_file: + with open(category_filename, 'r', encoding='utf-8') as category_file: category_str = category_file.read() except OSError: print('EX: unable to read category ' + category_filename) @@ -60,7 +60,7 @@ def get_hashtag_categories(base_dir: str, hashtag = catfile.split('.')[0] if len(hashtag) > MAX_TAG_LENGTH: continue - with open(category_filename, 'r') as fp_category: + with open(category_filename, 'r', encoding='utf-8') as fp_category: category_str = fp_category.read() if not category_str: @@ -120,7 +120,8 @@ def update_hashtag_categories(base_dir: str) -> None: # save a list of available categories for quick lookup try: - with open(category_list_filename, 'w+') as fp_category: + with open(category_list_filename, 'w+', + encoding='utf-8') as fp_category: fp_category.write(category_list_str) except OSError: print('EX: unable to write category ' + category_list_filename) @@ -171,7 +172,7 @@ def set_hashtag_category(base_dir: str, hashtag: str, category: str, category_written = False try: - with open(category_filename, 'w+') as fp_category: + with open(category_filename, 'w+', encoding='utf-8') as fp_category: fp_category.write(category) category_written = True except OSError as ex: diff --git a/city.py b/city.py index 4da36991a..512418e5d 100644 --- a/city.py +++ b/city.py @@ -211,7 +211,7 @@ def spoof_geolocation(base_dir: str, "", "", 0) cities = [] try: - with open(locations_filename, 'r') as loc_file: + with open(locations_filename, 'r', encoding='utf-8') as loc_file: cities = loc_file.readlines() except OSError: print('EX: unable to read locations ' + locations_filename) @@ -223,7 +223,7 @@ def spoof_geolocation(base_dir: str, if os.path.isfile(nogo_filename): nogo_list = [] try: - with open(nogo_filename, 'r') as nogo_file: + with open(nogo_filename, 'r', encoding='utf-8') as nogo_file: nogo_list = nogo_file.readlines() except OSError: print('EX: unable to read ' + nogo_filename) @@ -319,7 +319,7 @@ def get_spoofed_city(city: str, base_dir: str, city_filename = acct_dir(base_dir, nickname, domain) + '/city.txt' if os.path.isfile(city_filename): try: - with open(city_filename, 'r') as city_file: + with open(city_filename, 'r', encoding='utf-8') as city_file: city = city_file.read().replace('\n', '') except OSError: print('EX: unable to read ' + city_filename) diff --git a/content.py b/content.py index 5e5c1b769..21bed45d4 100644 --- a/content.py +++ b/content.py @@ -228,7 +228,7 @@ def dangerous_css(filename: str, allow_local_network_access: bool) -> bool: content = None try: - with open(filename, 'r') as css_file: + with open(filename, 'r', encoding='utf-8') as css_file: content = css_file.read().lower() except OSError: print('EX: unable to read css file ' + filename) @@ -281,7 +281,8 @@ def switch_words(base_dir: str, nickname: str, domain: str, content: str, if not os.path.isfile(switch_words_filename): return content try: - with open(switch_words_filename, 'r') as words_file: + with open(switch_words_filename, 'r', + encoding='utf-8') as words_file: rules = words_file.readlines() except OSError: print('EX: unable to read switches ' + switch_words_filename) @@ -376,7 +377,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None: common_emoji = None if os.path.isfile(common_emoji_filename): try: - with open(common_emoji_filename, 'r') as fp_emoji: + with open(common_emoji_filename, 'r', + encoding='utf-8') as fp_emoji: common_emoji = fp_emoji.readlines() except OSError: print('EX: unable to load common emoji file') @@ -400,7 +402,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None: new_common_emoji.append(str(1).zfill(16) + ' ' + emoji_content) new_common_emoji.sort(reverse=True) try: - with open(common_emoji_filename, 'w+') as fp_emoji: + with open(common_emoji_filename, 'w+', + encoding='utf-8') as fp_emoji: for line in new_common_emoji: fp_emoji.write(line + '\n') except OSError: @@ -409,7 +412,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None: else: line = str(1).zfill(16) + ' ' + emoji_content + '\n' try: - with open(common_emoji_filename, 'w+') as fp_emoji: + with open(common_emoji_filename, 'w+', + encoding='utf-8') as fp_emoji: fp_emoji.write(line) except OSError: print('EX: error writing common emoji 2') @@ -929,19 +933,19 @@ def remove_long_words(content: str, max_word_length: int, continue if 'https:' in word_str: continue - elif 'http:' in word_str: + if 'http:' in word_str: continue - elif 'i2p:' in word_str: + if 'i2p:' in word_str: continue - elif 'gnunet:' in word_str: + if 'gnunet:' in word_str: continue - elif 'dat:' in word_str: + if 'dat:' in word_str: continue - elif 'rad:' in word_str: + if 'rad:' in word_str: continue - elif 'hyper:' in word_str: + if 'hyper:' in word_str: continue - elif 'briar:' in word_str: + if 'briar:' in word_str: continue if '<' in word_str: replace_word = word_str.split('<', 1)[0] @@ -973,7 +977,7 @@ def _load_auto_tags(base_dir: str, nickname: str, domain: str) -> []: if not os.path.isfile(filename): return [] try: - with open(filename, 'r') as tags_file: + with open(filename, 'r', encoding='utf-8') as tags_file: return tags_file.readlines() except OSError: print('EX: unable to read auto tags ' + filename) @@ -1052,7 +1056,8 @@ def add_html_tags(base_dir: str, http_prefix: str, if os.path.isfile(following_filename): following = [] try: - with open(following_filename, 'r') as foll_file: + with open(following_filename, 'r', + encoding='utf-8') as foll_file: following = foll_file.readlines() except OSError: print('EX: unable to read ' + following_filename) @@ -1557,7 +1562,7 @@ def import_emoji(base_dir: str, import_filename: str, session) -> None: return emoji_dict = load_json(base_dir + '/emoji/default_emoji.json', 0, 1) added = 0 - with open(import_filename, "r") as fp_emoji: + with open(import_filename, "r", encoding='utf-8') as fp_emoji: lines = fp_emoji.readlines() for line in lines: url = line.split(', ')[0] @@ -1713,7 +1718,8 @@ def remove_script(content: str, log_filename: str, if os.path.isfile(log_filename): write_type = 'w+' try: - with open(log_filename, write_type) as fp_log: + with open(log_filename, write_type, + encoding='utf-8') as fp_log: fp_log.write(log_str) except OSError: print('EX: cannot append to svg script log') diff --git a/conversation.py b/conversation.py index 448d46e7f..8b1e697e6 100644 --- a/conversation.py +++ b/conversation.py @@ -43,15 +43,18 @@ def update_conversation(base_dir: str, nickname: str, domain: str, post_id = remove_id_ending(post_json_object['object']['id']) if not os.path.isfile(conversation_filename): try: - with open(conversation_filename, 'w+') as conv_file: + with open(conversation_filename, 'w+', + encoding='utf-8') as conv_file: conv_file.write(post_id + '\n') return True except OSError: print('EX: update_conversation ' + 'unable to write to ' + conversation_filename) - elif post_id + '\n' not in open(conversation_filename).read(): + elif post_id + '\n' not in open(conversation_filename, + encoding='utf-8').read(): try: - with open(conversation_filename, 'a+') as conv_file: + with open(conversation_filename, 'a+', + encoding='utf-8') as conv_file: conv_file.write(post_id + '\n') return True except OSError: @@ -72,7 +75,8 @@ def mute_conversation(base_dir: str, nickname: str, domain: str, if os.path.isfile(conversation_filename + '.muted'): return try: - with open(conversation_filename + '.muted', 'w+') as conv_file: + with open(conversation_filename + '.muted', 'w+', + encoding='utf-8') as conv_file: conv_file.write('\n') except OSError: print('EX: unable to write mute ' + conversation_filename) diff --git a/crawlers.py b/crawlers.py index fcca4d056..1c9bc72b5 100644 --- a/crawlers.py +++ b/crawlers.py @@ -60,7 +60,7 @@ def load_known_web_bots(base_dir: str) -> []: return [] crawlers_str = None try: - with open(known_bots_filename, 'r') as fp_crawlers: + with open(known_bots_filename, 'r', encoding='utf-8') as fp_crawlers: crawlers_str = fp_crawlers.read() except OSError: print('EX: unable to load web bots from ' + @@ -88,7 +88,7 @@ def _save_known_web_bots(base_dir: str, known_bots: []) -> bool: for crawler in known_bots: known_bots_str += crawler.strip() + '\n' try: - with open(known_bots_filename, 'w+') as fp_crawlers: + with open(known_bots_filename, 'w+', encoding='utf-8') as fp_crawlers: fp_crawlers.write(known_bots_str) except OSError: print("EX: unable to save known web bots to " + diff --git a/daemon.py b/daemon.py index 22132280e..da56b2bec 100644 --- a/daemon.py +++ b/daemon.py @@ -510,7 +510,7 @@ class PubServer(BaseHTTPRequestHandler): if os.path.isfile(votes_filename): # have we already voted on this? - if message_id in open(votes_filename).read(): + if message_id in open(votes_filename, encoding='utf-8').read(): print('Already voted on message ' + message_id) return @@ -580,7 +580,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.debug) # record the vote try: - with open(votes_filename, 'a+') as votes_file: + with open(votes_filename, 'a+', + encoding='utf-8') as votes_file: votes_file.write(message_id + '\n') except OSError: print('EX: unable to write vote ' + @@ -892,7 +893,8 @@ class PubServer(BaseHTTPRequestHandler): etag = None if os.path.isfile(media_filename + '.etag'): try: - with open(media_filename + '.etag', 'r') as efile: + with open(media_filename + '.etag', 'r', + encoding='utf-8') as efile: etag = efile.read() except OSError: print('EX: _set_headers_etag ' + @@ -900,7 +902,8 @@ class PubServer(BaseHTTPRequestHandler): if not etag: etag = md5(data).hexdigest() # nosec try: - with open(media_filename + '.etag', 'w+') as efile: + with open(media_filename + '.etag', 'w+', + encoding='utf-8') as efile: efile.write(etag) except OSError: print('EX: _set_headers_etag ' + @@ -926,7 +929,8 @@ class PubServer(BaseHTTPRequestHandler): # load the etag from file curr_etag = '' try: - with open(media_filename + '.etag', 'r') as efile: + with open(media_filename + '.etag', 'r', + encoding='utf-8') as efile: curr_etag = efile.read() except OSError: print('EX: _etag_exists unable to read ' + @@ -1953,9 +1957,9 @@ class PubServer(BaseHTTPRequestHandler): # to be authorized to use an account you don't own if '/' + nickname + '/' in self.path: return True - elif '/' + nickname + '?' in self.path: + if '/' + nickname + '?' in self.path: return True - elif self.path.endswith('/' + nickname): + if self.path.endswith('/' + nickname): return True if self.server.debug: print('AUTH: nickname ' + nickname + @@ -2128,14 +2132,16 @@ class PubServer(BaseHTTPRequestHandler): salt = create_password(32) if os.path.isfile(salt_filename): try: - with open(salt_filename, 'r') as fp_salt: + with open(salt_filename, 'r', + encoding='utf-8') as fp_salt: salt = fp_salt.read() except OSError as ex: print('EX: Unable to read salt for ' + login_nickname + ' ' + str(ex)) else: try: - with open(salt_filename, 'w+') as fp_salt: + with open(salt_filename, 'w+', + encoding='utf-8') as fp_salt: fp_salt.write(salt) except OSError as ex: print('EX: Unable to save salt for ' + @@ -2149,7 +2155,8 @@ class PubServer(BaseHTTPRequestHandler): base_dir + '/accounts/' + \ login_handle + '/.token' try: - with open(token_filename, 'w+') as fp_tok: + with open(token_filename, 'w+', + encoding='utf-8') as fp_tok: fp_tok.write(token) except OSError as ex: print('EX: Unable to save token for ' + @@ -2960,7 +2967,8 @@ class PubServer(BaseHTTPRequestHandler): feat_filename = features_blocked_filename feat_written = False try: - with open(feat_filename, 'w+') as nofile: + with open(feat_filename, 'w+', + encoding='utf-8') as nofile: nofile.write('\n') feat_written = True except OSError as ex: @@ -3004,7 +3012,8 @@ class PubServer(BaseHTTPRequestHandler): if os.path.isdir(account_dir): nw_filename = newswire_mod_filename try: - with open(nw_filename, 'w+') as modfile: + with open(nw_filename, 'w+', + encoding='utf-8') as modfile: modfile.write('\n') except OSError: print('EX: unable to write ' + nw_filename) @@ -4781,7 +4790,8 @@ class PubServer(BaseHTTPRequestHandler): links_str += '\n' links_str += fields['newColLink'] + '\n' try: - with open(links_filename, 'w+') as linksfile: + with open(links_filename, 'w+', + encoding='utf-8') as linksfile: linksfile.write(links_str) except OSError: print('EX: _links_update unable to write ' + @@ -4791,7 +4801,8 @@ class PubServer(BaseHTTPRequestHandler): # the text area is empty but there is a new link added links_str = fields['newColLink'] + '\n' try: - with open(links_filename, 'w+') as linksfile: + with open(links_filename, 'w+', + encoding='utf-8') as linksfile: linksfile.write(links_str) except OSError: print('EX: _links_update unable to write ' + @@ -4812,7 +4823,8 @@ class PubServer(BaseHTTPRequestHandler): if not dangerous_markup(about_str, allow_local_network_access): try: - with open(about_filename, 'w+') as aboutfile: + with open(about_filename, 'w+', + encoding='utf-8') as aboutfile: aboutfile.write(about_str) except OSError: print('EX: unable to write about ' + @@ -4830,7 +4842,8 @@ class PubServer(BaseHTTPRequestHandler): if not dangerous_markup(tos_str, allow_local_network_access): try: - with open(tos_filename, 'w+') as tosfile: + with open(tos_filename, 'w+', + encoding='utf-8') as tosfile: tosfile.write(tos_str) except OSError: print('EX: unable to write TOS ' + tos_filename) @@ -5024,7 +5037,8 @@ class PubServer(BaseHTTPRequestHandler): newswire_str += '\n' newswire_str += fields['newNewswireFeed'] + '\n' try: - with open(newswire_filename, 'w+') as newsfile: + with open(newswire_filename, 'w+', + encoding='utf-8') as newsfile: newsfile.write(newswire_str) except OSError: print('EX: unable to write ' + newswire_filename) @@ -5033,7 +5047,8 @@ class PubServer(BaseHTTPRequestHandler): # the text area is empty but there is a new feed added newswire_str = fields['newNewswireFeed'] + '\n' try: - with open(newswire_filename, 'w+') as newsfile: + with open(newswire_filename, 'w+', + encoding='utf-8') as newsfile: newsfile.write(newswire_str) except OSError: print('EX: unable to write ' + newswire_filename) @@ -5052,7 +5067,8 @@ class PubServer(BaseHTTPRequestHandler): 'news@' + domain + '/filters.txt' if fields.get('filteredWordsNewswire'): try: - with open(filter_newswire_filename, 'w+') as filterfile: + with open(filter_newswire_filename, 'w+', + encoding='utf-8') as filterfile: filterfile.write(fields['filteredWordsNewswire']) except OSError: print('EX: unable to write ' + filter_newswire_filename) @@ -5069,7 +5085,8 @@ class PubServer(BaseHTTPRequestHandler): base_dir + '/accounts/hashtagrules.txt' if fields.get('hashtagRulesList'): try: - with open(hashtag_rules_filename, 'w+') as rulesfile: + with open(hashtag_rules_filename, 'w+', + encoding='utf-8') as rulesfile: rulesfile.write(fields['hashtagRulesList']) except OSError: print('EX: unable to write ' + hashtag_rules_filename) @@ -5088,7 +5105,8 @@ class PubServer(BaseHTTPRequestHandler): if not newswire_trusted.endswith('\n'): newswire_trusted += '\n' try: - with open(newswire_tusted_filename, 'w+') as trustfile: + with open(newswire_tusted_filename, 'w+', + encoding='utf-8') as trustfile: trustfile.write(newswire_trusted) except OSError: print('EX: unable to write ' + newswire_tusted_filename) @@ -5188,7 +5206,8 @@ class PubServer(BaseHTTPRequestHandler): # save citations dates, so that they can be added when # reloading the newblog screen try: - with open(citations_filename, 'w+') as citfile: + with open(citations_filename, 'w+', + encoding='utf-8') as citfile: citfile.write(citations_str) except OSError: print('EX: unable to write ' + citations_filename) @@ -6315,8 +6334,8 @@ class PubServer(BaseHTTPRequestHandler): # if the list was given as comma separated mods = fields['moderators'].split(',') try: - with open(moderators_file, - 'w+') as modfile: + with open(moderators_file, 'w+', + encoding='utf-8') as modfile: for mod_nick in mods: mod_nick = mod_nick.strip() mod_dir = base_dir + \ @@ -6343,8 +6362,8 @@ class PubServer(BaseHTTPRequestHandler): # nicknames on separate lines mods = fields['moderators'].split('\n') try: - with open(moderators_file, - 'w+') as modfile: + with open(moderators_file, 'w+', + encoding='utf-8') as modfile: for mod_nick in mods: mod_nick = mod_nick.strip() mod_dir = \ @@ -6447,8 +6466,8 @@ class PubServer(BaseHTTPRequestHandler): # if the list was given as comma separated eds = fields['counselors'].split(',') try: - with open(counselors_file, - 'w+') as edfile: + with open(counselors_file, 'w+', + encoding='utf-8') as edfile: for ed_nick in eds: ed_nick = ed_nick.strip() ed_dir = base_dir + \ @@ -6475,8 +6494,8 @@ class PubServer(BaseHTTPRequestHandler): # nicknames on separate lines eds = fields['counselors'].split('\n') try: - with open(counselors_file, - 'w+') as edfile: + with open(counselors_file, 'w+', + encoding='utf-8') as edfile: for ed_nick in eds: ed_nick = ed_nick.strip() ed_dir = \ @@ -6515,7 +6534,8 @@ class PubServer(BaseHTTPRequestHandler): # if the list was given as comma separated eds = fields['artists'].split(',') try: - with open(artists_file, 'w+') as edfil: + with open(artists_file, 'w+', + encoding='utf-8') as edfil: for ed_nick in eds: ed_nick = ed_nick.strip() ed_dir = base_dir + \ @@ -6540,7 +6560,8 @@ class PubServer(BaseHTTPRequestHandler): # nicknames on separate lines eds = fields['artists'].split('\n') try: - with open(artists_file, 'w+') as edfil: + with open(artists_file, 'w+', + encoding='utf-8') as edfil: for ed_nick in eds: ed_nick = ed_nick.strip() ed_dir = \ @@ -6655,7 +6676,8 @@ class PubServer(BaseHTTPRequestHandler): # initial default setting created via # the welcome screen try: - with open(follow_dms_filename, 'w+') as ffile: + with open(follow_dms_filename, 'w+', + encoding='utf-8') as ffile: ffile.write('\n') except OSError: print('EX: unable to write follow DMs ' + @@ -6667,8 +6689,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['followDMs'] == 'on': follow_dms_active = True try: - with open(follow_dms_filename, - 'w+') as ffile: + with open(follow_dms_filename, 'w+', + encoding='utf-8') as ffile: ffile.write('\n') except OSError: print('EX: unable to write follow DMs 2 ' + @@ -6691,8 +6713,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['removeTwitter'] == 'on': remove_twitter_active = True try: - with open(remove_twitter_filename, - 'w+') as rfile: + with open(remove_twitter_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write remove twitter ' + @@ -6718,7 +6740,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['hideLikeButton'] == 'on': hide_like_button_active = True try: - with open(hide_like_button_file, 'w+') as rfil: + with open(hide_like_button_file, 'w+', + encoding='utf-8') as rfil: rfil.write('\n') except OSError: print('EX: unable to write hide like ' + @@ -6752,8 +6775,8 @@ class PubServer(BaseHTTPRequestHandler): if fields['hideReactionButton'] == 'on': hide_reaction_button_active = True try: - with open(hide_reaction_button_file, - 'w+') as rfile: + with open(hide_reaction_button_file, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write hide reaction ' + @@ -6785,8 +6808,8 @@ class PubServer(BaseHTTPRequestHandler): bold_reading = True self.server.bold_reading[nickname] = True try: - with open(bold_reading_filename, - 'w+') as rfile: + with open(bold_reading_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write bold reading ' + @@ -6806,7 +6829,8 @@ class PubServer(BaseHTTPRequestHandler): if on_final_welcome_screen: # default setting from welcome screen try: - with open(notify_likes_filename, 'w+') as rfile: + with open(notify_likes_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write notify likes ' + @@ -6819,8 +6843,8 @@ class PubServer(BaseHTTPRequestHandler): not hide_like_button_active: notify_likes_active = True try: - with open(notify_likes_filename, - 'w+') as rfile: + with open(notify_likes_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write notify likes ' + @@ -6841,7 +6865,8 @@ class PubServer(BaseHTTPRequestHandler): # default setting from welcome screen notify_react_filename = notify_reactions_filename try: - with open(notify_react_filename, 'w+') as rfile: + with open(notify_react_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write notify reactions ' + @@ -6854,8 +6879,8 @@ class PubServer(BaseHTTPRequestHandler): not hide_reaction_button_active: notify_reactions_active = True try: - with open(notify_reactions_filename, - 'w+') as rfile: + with open(notify_reactions_filename, 'w+', + encoding='utf-8') as rfile: rfile.write('\n') except OSError: print('EX: unable to write ' + @@ -6945,7 +6970,8 @@ class PubServer(BaseHTTPRequestHandler): '/filters.txt' if fields.get('filteredWords'): try: - with open(filter_filename, 'w+') as filterfile: + with open(filter_filename, 'w+', + encoding='utf-8') as filterfile: filterfile.write(fields['filteredWords']) except OSError: print('EX: unable to write filter ' + @@ -6965,7 +6991,8 @@ class PubServer(BaseHTTPRequestHandler): '/filters_bio.txt' if fields.get('filteredWordsBio'): try: - with open(filter_bio_filename, 'w+') as filterfile: + with open(filter_bio_filename, 'w+', + encoding='utf-8') as filterfile: filterfile.write(fields['filteredWordsBio']) except OSError: print('EX: unable to write bio filter ' + @@ -6985,7 +7012,8 @@ class PubServer(BaseHTTPRequestHandler): '/replacewords.txt' if fields.get('switchwords'): try: - with open(switch_filename, 'w+') as switchfile: + with open(switch_filename, 'w+', + encoding='utf-8') as switchfile: switchfile.write(fields['switchwords']) except OSError: print('EX: unable to write switches ' + @@ -7005,7 +7033,8 @@ class PubServer(BaseHTTPRequestHandler): '/autotags.txt' if fields.get('autoTags'): try: - with open(auto_tags_filename, 'w+') as autofile: + with open(auto_tags_filename, 'w+', + encoding='utf-8') as autofile: autofile.write(fields['autoTags']) except OSError: print('EX: unable to write auto tags ' + @@ -7025,7 +7054,8 @@ class PubServer(BaseHTTPRequestHandler): '/autocw.txt' if fields.get('autoCW'): try: - with open(auto_cw_filename, 'w+') as auto_cw_file: + with open(auto_cw_filename, 'w+', + encoding='utf-8') as auto_cw_file: auto_cw_file.write(fields['autoCW']) except OSError: print('EX: unable to write auto CW ' + @@ -7045,7 +7075,8 @@ class PubServer(BaseHTTPRequestHandler): '/blocking.txt' if fields.get('blocked'): try: - with open(blocked_filename, 'w+') as blockedfile: + with open(blocked_filename, 'w+', + encoding='utf-8') as blockedfile: blockedfile.write(fields['blocked']) except OSError: print('EX: unable to write blocked accounts ' + @@ -7067,8 +7098,8 @@ class PubServer(BaseHTTPRequestHandler): '/dmAllowedinstances.txt' if fields.get('dmAllowedInstances'): try: - with open(dm_allowed_instances_filename, - 'w+') as afile: + with open(dm_allowed_instances_filename, 'w+', + encoding='utf-8') as afile: afile.write(fields['dmAllowedInstances']) except OSError: print('EX: unable to write allowed DM instances ' + @@ -7090,7 +7121,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('allowedInstances'): inst_filename = allowed_instances_filename try: - with open(inst_filename, 'w+') as afile: + with open(inst_filename, 'w+', + encoding='utf-8') as afile: afile.write(fields['allowedInstances']) except OSError: print('EX: unable to write allowed instances ' + @@ -7173,8 +7205,8 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('ptInstances'): self.server.peertube_instances.clear() try: - with open(peertube_instances_file, - 'w+') as afile: + with open(peertube_instances_file, 'w+', + encoding='utf-8') as afile: afile.write(fields['ptInstances']) except OSError: print('EX: unable to write peertube ' + @@ -7205,7 +7237,8 @@ class PubServer(BaseHTTPRequestHandler): '/gitprojects.txt' if fields.get('gitProjects'): try: - with open(git_projects_filename, 'w+') as afile: + with open(git_projects_filename, 'w+', + encoding='utf-8') as afile: afile.write(fields['gitProjects'].lower()) except OSError: print('EX: unable to write git ' + @@ -7643,9 +7676,7 @@ class PubServer(BaseHTTPRequestHandler): return msg = \ - html_blog_page_rss2(authorized, - curr_session, - base_dir, + html_blog_page_rss2(base_dir, http_prefix, self.server.translate, nickname, @@ -7698,9 +7729,7 @@ class PubServer(BaseHTTPRequestHandler): nickname = acct.split('@')[0] domain = acct.split('@')[1] msg += \ - html_blog_page_rss2(authorized, - curr_session, - base_dir, + html_blog_page_rss2(base_dir, http_prefix, self.server.translate, nickname, @@ -7807,8 +7836,7 @@ class PubServer(BaseHTTPRequestHandler): path + ' ' + calling_domain) self._404() - def _get_rss3feed(self, authorized: bool, - calling_domain: str, path: str, + def _get_rss3feed(self, calling_domain: str, path: str, base_dir: str, http_prefix: str, domain: str, port: int, proxy_type: str, getreq_start_time, @@ -7829,10 +7857,7 @@ class PubServer(BaseHTTPRequestHandler): self._404() return msg = \ - html_blog_page_rss3(authorized, - curr_session, - base_dir, http_prefix, - self.server.translate, + html_blog_page_rss3(base_dir, http_prefix, nickname, domain, port, MAX_POSTS_IN_RSS_FEED, 1, system_language) @@ -7910,8 +7935,7 @@ class PubServer(BaseHTTPRequestHandler): actor_json = \ get_person_from_cache(base_dir, options_actor, - self.server.person_cache, - True) + self.server.person_cache) if actor_json: if actor_json.get('movedTo'): moved_to = actor_json['movedTo'] @@ -9966,20 +9990,10 @@ class PubServer(BaseHTTPRequestHandler): local_actor_url(http_prefix, self.post_to_nickname, domain_full) cc_list = [] bookmark_post(self.server.recent_posts_cache, - curr_session, - base_dir, - self.server.federation_list, - self.post_to_nickname, - domain, port, - cc_list, - http_prefix, - bookmark_url, bookmark_actor, False, - self.server.send_threads, - self.server.postLog, - self.server.person_cache, - self.server.cached_webfingers, - self.server.debug, - self.server.project_version) + base_dir, self.server.federation_list, + self.post_to_nickname, domain, port, + cc_list, http_prefix, bookmark_url, bookmark_actor, + self.server.debug) # clear the icon from the cache so that it gets updated if self.server.iconsCache.get('bookmark.png'): del self.server.iconsCache['bookmark.png'] @@ -10122,20 +10136,10 @@ class PubServer(BaseHTTPRequestHandler): local_actor_url(http_prefix, self.post_to_nickname, domain_full) cc_list = [] undo_bookmark_post(self.server.recent_posts_cache, - curr_session, - base_dir, - self.server.federation_list, + base_dir, self.server.federation_list, self.post_to_nickname, - domain, port, - cc_list, - http_prefix, - bookmark_url, undo_actor, False, - self.server.send_threads, - self.server.postLog, - self.server.person_cache, - self.server.cached_webfingers, - debug, - self.server.project_version) + domain, port, cc_list, http_prefix, + bookmark_url, undo_actor, debug) # clear the icon from the cache so that it gets updated if self.server.iconsCache.get('bookmark_inactive.png'): del self.server.iconsCache['bookmark_inactive.png'] @@ -11161,7 +11165,7 @@ class PubServer(BaseHTTPRequestHandler): return True ssml_str = None try: - with open(ssml_filename, 'r') as fp_ssml: + with open(ssml_filename, 'r', encoding='utf-8') as fp_ssml: ssml_str = fp_ssml.read() except OSError: pass @@ -14365,12 +14369,12 @@ class PubServer(BaseHTTPRequestHandler): """ image_extensions = get_image_extensions() for ext in image_extensions: - for bg in ('follow', 'options', 'login', 'welcome'): + for bg_im in ('follow', 'options', 'login', 'welcome'): # follow screen background image - if path.endswith('/' + bg + '-background.' + ext): + if path.endswith('/' + bg_im + '-background.' + ext): bg_filename = \ base_dir + '/accounts/' + \ - bg + '-background.' + ext + bg_im + '-background.' + ext if os.path.isfile(bg_filename): if self._etag_exists(bg_filename): # The file has not changed @@ -15662,8 +15666,7 @@ class PubServer(BaseHTTPRequestHandler): # RSS 3.0 if self.path.startswith('/blog/') and \ self.path.endswith('/rss.txt'): - self._get_rss3feed(authorized, - calling_domain, self.path, + self._get_rss3feed(calling_domain, self.path, self.server.base_dir, self.server.http_prefix, self.server.domain, @@ -17532,13 +17535,13 @@ class PubServer(BaseHTTPRequestHandler): in_reply_to_url = self.path.split('?replyto=')[1] if '?' in in_reply_to_url: mentions_list = in_reply_to_url.split('?') - for m in mentions_list: - if m.startswith('mention='): - reply_handle = m.replace('mention=', '') + for ment in mentions_list: + if ment.startswith('mention='): + reply_handle = ment.replace('mention=', '') if reply_handle not in reply_to_list: reply_to_list.append(reply_handle) - if m.startswith('page='): - reply_page_str = m.replace('page=', '') + if ment.startswith('page='): + reply_page_str = ment.replace('page=', '') if reply_page_str.isdigit(): reply_page_number = int(reply_page_str) # if m.startswith('actor='): @@ -17553,13 +17556,13 @@ class PubServer(BaseHTTPRequestHandler): in_reply_to_url = self.path.split('?replyunlisted=')[1] if '?' in in_reply_to_url: mentions_list = in_reply_to_url.split('?') - for m in mentions_list: - if m.startswith('mention='): - reply_handle = m.replace('mention=', '') + for ment in mentions_list: + if ment.startswith('mention='): + reply_handle = ment.replace('mention=', '') if reply_handle not in reply_to_list: reply_to_list.append(reply_handle) - if m.startswith('page='): - reply_page_str = m.replace('page=', '') + if ment.startswith('page='): + reply_page_str = ment.replace('page=', '') if reply_page_str.isdigit(): reply_page_number = int(reply_page_str) in_reply_to_url = mentions_list[0] @@ -17665,7 +17668,6 @@ class PubServer(BaseHTTPRequestHandler): msg = html_edit_blog(self.server.media_instance, self.server.translate, self.server.base_dir, - self.server.http_prefix, self.path, reply_page_number, nickname, self.server.domain, post_url, @@ -18830,7 +18832,8 @@ class PubServer(BaseHTTPRequestHandler): acct_dir(self.server.base_dir, nickname, self.server.domain) + '/.lastUsed' try: - with open(last_used_filename, 'w+') as lastfile: + with open(last_used_filename, 'w+', + encoding='utf-8') as lastfile: lastfile.write(str(int(time.time()))) except OSError: print('EX: _receive_new_post_process unable to write ' + diff --git a/desktop_client.py b/desktop_client.py index 8bbbbc5ca..9407a72e1 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -169,12 +169,14 @@ def _mark_post_as_read(actor: str, post_id: str, post_category: str) -> None: read_posts_dir = home_dir + '/.config/epicyon/' + handle read_posts_filename = read_posts_dir + '/' + post_category + '.txt' if os.path.isfile(read_posts_filename): - if post_id in open(read_posts_filename).read(): + if post_id in open(read_posts_filename, + encoding='utf-8').read(): return try: # prepend to read posts file post_id += '\n' - with open(read_posts_filename, 'r+') as read_file: + with open(read_posts_filename, 'r+', + encoding='utf-8') as read_file: content = read_file.read() if post_id not in content: read_file.seek(0, 0) @@ -182,7 +184,7 @@ def _mark_post_as_read(actor: str, post_id: str, post_category: str) -> None: except Exception as ex: print('EX: Failed to mark post as read' + str(ex)) else: - with open(read_posts_filename, 'w+') as read_file: + with open(read_posts_filename, 'w+', encoding='utf-8') as read_file: read_file.write(post_id + '\n') @@ -292,7 +294,7 @@ def _desktop_show_banner() -> None: banner_filename = 'theme/' + banner_theme + '/banner.txt' if not os.path.isfile(banner_filename): return - with open(banner_filename, 'r') as banner_file: + with open(banner_filename, 'r', encoding='utf-8') as banner_file: banner = banner_file.read() if banner: print(banner + '\n') @@ -1325,7 +1327,7 @@ def _desktop_show_follow_requests(follow_requests_json: {}, def _desktop_show_following(following_json: {}, translate: {}, page_number: int, indent: str, - followType='following') -> None: + follow_type: str = 'following') -> None: """Shows a page of accounts followed """ if not isinstance(following_json, dict): @@ -1335,9 +1337,9 @@ def _desktop_show_following(following_json: {}, translate: {}, if not following_json['orderedItems']: return print('') - if followType == 'following': + if follow_type == 'following': print(indent + 'Following page ' + str(page_number)) - elif followType == 'followers': + elif follow_type == 'followers': print(indent + 'Followers page ' + str(page_number)) print('') for item in following_json['orderedItems']: @@ -2198,7 +2200,7 @@ def run_desktop_client(base_dir: str, proxy_type: str, http_prefix: str, post_json_object, nickname, password, domain, port, - http_prefix, post_id, + http_prefix, cached_webfingers, person_cache, True, __version__, diff --git a/epicyon.py b/epicyon.py index 1a3dddbaa..cb5b1786f 100644 --- a/epicyon.py +++ b/epicyon.py @@ -1006,7 +1006,7 @@ def _command_options() -> None: __version__, argb.language, signing_priv_key_pem) try: - with open('socnet.dot', 'w+') as fp_soc: + with open('socnet.dot', 'w+', encoding='utf-8') as fp_soc: fp_soc.write(dot_graph) print('Saved to socnet.dot') except OSError: @@ -1453,7 +1453,8 @@ def _command_options() -> None: approve_follows_filename = accounts_dir + '/followrequests.txt' approve_ctr = 0 if os.path.isfile(approve_follows_filename): - with open(approve_follows_filename, 'r') as approvefile: + with open(approve_follows_filename, 'r', + encoding='utf-8') as approvefile: for approve in approvefile: print(approve.replace('\n', '').replace('\r', '')) approve_ctr += 1 diff --git a/filters.py b/filters.py index 77968a47d..97ca31432 100644 --- a/filters.py +++ b/filters.py @@ -16,10 +16,11 @@ def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool: """ filters_filename = acct_dir(base_dir, nickname, domain) + '/filters.txt' if os.path.isfile(filters_filename): - if words in open(filters_filename).read(): + if words in open(filters_filename, encoding='utf-8').read(): return False try: - with open(filters_filename, 'a+') as filters_file: + with open(filters_filename, 'a+', + encoding='utf-8') as filters_file: filters_file.write(words + '\n') except OSError: print('EX: unable to append filters ' + filters_filename) @@ -36,10 +37,10 @@ def add_global_filter(base_dir: str, words: str) -> bool: return False filters_filename = base_dir + '/accounts/filters.txt' if os.path.isfile(filters_filename): - if words in open(filters_filename).read(): + if words in open(filters_filename, encoding='utf-8').read(): return False try: - with open(filters_filename, 'a+') as filters_file: + with open(filters_filename, 'a+', encoding='utf-8') as filters_file: filters_file.write(words + '\n') except OSError: print('EX: unable to append filters ' + filters_filename) @@ -53,12 +54,12 @@ def remove_filter(base_dir: str, nickname: str, domain: str, filters_filename = acct_dir(base_dir, nickname, domain) + '/filters.txt' if not os.path.isfile(filters_filename): return False - if words not in open(filters_filename).read(): + if words not in open(filters_filename, encoding='utf-8').read(): return False new_filters_filename = filters_filename + '.new' try: - with open(filters_filename, 'r') as fp_filt: - with open(new_filters_filename, 'w+') as fpnew: + with open(filters_filename, 'r', encoding='utf-8') as fp_filt: + with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: for line in fp_filt: line = line.replace('\n', '') if line != words: @@ -78,12 +79,12 @@ def remove_global_filter(base_dir: str, words: str) -> bool: filters_filename = base_dir + '/accounts/filters.txt' if not os.path.isfile(filters_filename): return False - if words not in open(filters_filename).read(): + if words not in open(filters_filename, encoding='utf-8').read(): return False new_filters_filename = filters_filename + '.new' try: - with open(filters_filename, 'r') as fp_filt: - with open(new_filters_filename, 'w+') as fpnew: + with open(filters_filename, 'r', encoding='utf-8') as fp_filt: + with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: for line in fp_filt: line = line.replace('\n', '') if line != words: @@ -118,7 +119,7 @@ def _is_filtered_base(filename: str, content: str) -> bool: return False try: - with open(filename, 'r') as fp_filt: + with open(filename, 'r', encoding='utf-8') as fp_filt: for line in fp_filt: filter_str = line.replace('\n', '').replace('\r', '') if not filter_str: diff --git a/follow.py b/follow.py index 03e53cd15..a9b482f71 100644 --- a/follow.py +++ b/follow.py @@ -56,7 +56,8 @@ def create_initial_last_seen(base_dir: str, http_prefix: str) -> None: os.mkdir(last_seen_dir) following_handles = [] try: - with open(following_filename, 'r') as fp_foll: + with open(following_filename, 'r', + encoding='utf-8') as fp_foll: following_handles = fp_foll.readlines() except OSError: print('EX: create_initial_last_seen ' + following_filename) @@ -75,7 +76,8 @@ def create_initial_last_seen(base_dir: str, http_prefix: str) -> None: last_seen_dir + '/' + actor.replace('/', '#') + '.txt' if not os.path.isfile(last_seen_filename): try: - with open(last_seen_filename, 'w+') as fp_last: + with open(last_seen_filename, 'w+', + encoding='utf-8') as fp_last: fp_last.write(str(100)) except OSError: print('EX: create_initial_last_seen 2 ' + @@ -92,7 +94,8 @@ def _pre_approved_follower(base_dir: str, account_dir = base_dir + '/accounts/' + handle approved_filename = account_dir + '/approved.txt' if os.path.isfile(approved_filename): - if approve_handle in open(approved_filename).read(): + if approve_handle in open(approved_filename, + encoding='utf-8').read(): return True return False @@ -112,7 +115,8 @@ def _remove_from_follow_base(base_dir: str, ' to remove ' + handle + ' from') return accept_deny_actor = None - if accept_or_deny_handle not in open(approve_follows_filename).read(): + if accept_or_deny_handle not in open(approve_follows_filename, + encoding='utf-8').read(): # is this stored in the file as an actor rather than a handle? accept_deny_nickname = accept_or_deny_handle.split('@')[0] accept_deny_domain = accept_or_deny_handle.split('@')[1] @@ -123,14 +127,17 @@ def _remove_from_follow_base(base_dir: str, for users_name in users_paths: accept_deny_actor = \ '://' + accept_deny_domain + users_name + accept_deny_nickname - if accept_deny_actor in open(approve_follows_filename).read(): + if accept_deny_actor in open(approve_follows_filename, + encoding='utf-8').read(): actor_found = True break if not actor_found: return try: - with open(approve_follows_filename + '.new', 'w+') as approvefilenew: - with open(approve_follows_filename, 'r') as approvefile: + with open(approve_follows_filename + '.new', 'w+', + encoding='utf-8') as approvefilenew: + with open(approve_follows_filename, 'r', + encoding='utf-8') as approvefile: if not accept_deny_actor: for approve_handle in approvefile: accept_deny_handle = accept_or_deny_handle @@ -179,7 +186,7 @@ def is_following_actor(base_dir: str, return False if actor.startswith('@'): actor = actor[1:] - if actor.lower() in open(following_file).read().lower(): + if actor.lower() in open(following_file, encoding='utf-8').read().lower(): return True following_nickname = get_nickname_from_actor(actor) if not following_nickname: @@ -189,7 +196,8 @@ def is_following_actor(base_dir: str, following_handle = \ get_full_domain(following_nickname + '@' + following_domain, following_port) - if following_handle.lower() in open(following_file).read().lower(): + if following_handle.lower() in open(following_file, + encoding='utf-8').read().lower(): return True return False @@ -232,7 +240,7 @@ def get_follower_domains(base_dir: str, nickname: str, domain: str) -> []: lines = [] try: - with open(followers_file, 'r') as fp_foll: + with open(followers_file, 'r', encoding='utf-8') as fp_foll: lines = fp_foll.readlines() except OSError: print('EX: get_follower_domains ' + followers_file) @@ -269,7 +277,7 @@ def is_follower_of_person(base_dir: str, nickname: str, domain: str, followers_str = '' try: - with open(followers_file, 'r') as fp_foll: + with open(followers_file, 'r', encoding='utf-8') as fp_foll: followers_str = fp_foll.read() except OSError: print('EX: is_follower_of_person ' + followers_file) @@ -309,20 +317,21 @@ def unfollow_account(base_dir: str, nickname: str, domain: str, print('DEBUG: follow file ' + filename + ' was not found') return False handle_to_unfollow_lower = handle_to_unfollow.lower() - if handle_to_unfollow_lower not in open(filename).read().lower(): + if handle_to_unfollow_lower not in open(filename, + encoding='utf-8').read().lower(): if debug: print('DEBUG: handle to unfollow ' + handle_to_unfollow + ' is not in ' + filename) return lines = [] try: - with open(filename, 'r') as fp_unfoll: + with open(filename, 'r', encoding='utf-8') as fp_unfoll: lines = fp_unfoll.readlines() except OSError: print('EX: unfollow_account ' + filename) if lines: try: - with open(filename, 'w+') as fp_unfoll: + with open(filename, 'w+', encoding='utf-8') as fp_unfoll: for line in lines: check_handle = line.strip("\n").strip("\r").lower() if check_handle not in (handle_to_unfollow_lower, @@ -336,15 +345,17 @@ def unfollow_account(base_dir: str, nickname: str, domain: str, unfollowed_filename = base_dir + '/accounts/' + handle + '/unfollowed.txt' if os.path.isfile(unfollowed_filename): if handle_to_unfollow_lower not in \ - open(unfollowed_filename).read().lower(): + open(unfollowed_filename, encoding='utf-8').read().lower(): try: - with open(unfollowed_filename, 'a+') as fp_unfoll: + with open(unfollowed_filename, 'a+', + encoding='utf-8') as fp_unfoll: fp_unfoll.write(handle_to_unfollow + '\n') except OSError: print('EX: unable to append ' + unfollowed_filename) else: try: - with open(unfollowed_filename, 'w+') as fp_unfoll: + with open(unfollowed_filename, 'w+', + encoding='utf-8') as fp_unfoll: fp_unfoll.write(handle_to_unfollow + '\n') except OSError: print('EX: unable to write ' + unfollowed_filename) @@ -400,7 +411,7 @@ def _get_no_of_follows(base_dir: str, nickname: str, domain: str, ctr = 0 lines = [] try: - with open(filename, 'r') as fp_foll: + with open(filename, 'r', encoding='utf-8') as fp_foll: lines = fp_foll.readlines() except OSError: print('EX: _get_no_of_follows ' + filename) @@ -517,7 +528,7 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str, total_ctr = 0 lines = [] try: - with open(filename, 'r') as fp_foll: + with open(filename, 'r', encoding='utf-8') as fp_foll: lines = fp_foll.readlines() except OSError: print('EX: get_following_feed ' + filename) @@ -606,7 +617,8 @@ def no_of_follow_requests(base_dir: str, ctr = 0 lines = [] try: - with open(approve_follows_filename, 'r') as fp_approve: + with open(approve_follows_filename, 'r', + encoding='utf-8') as fp_approve: lines = fp_approve.readlines() except OSError: print('EX: no_of_follow_requests ' + approve_follows_filename) @@ -650,7 +662,8 @@ def store_follow_request(base_dir: str, followers_str = '' try: - with open(followers_filename, 'r') as fp_foll: + with open(followers_filename, 'r', + encoding='utf-8') as fp_foll: followers_str = fp_foll.read() except OSError: print('EX: store_follow_request ' + followers_filename) @@ -675,7 +688,8 @@ def store_follow_request(base_dir: str, # should this follow be denied? deny_follows_filename = accounts_dir + '/followrejects.txt' if os.path.isfile(deny_follows_filename): - if approve_handle in open(deny_follows_filename).read(): + if approve_handle in open(deny_follows_filename, + encoding='utf-8').read(): remove_from_follow_requests(base_dir, nickname_to_follow, domain_to_follow, approve_handle, debug) @@ -694,9 +708,11 @@ def store_follow_request(base_dir: str, approve_handle = '!' + approve_handle if os.path.isfile(approve_follows_filename): - if approve_handle not in open(approve_follows_filename).read(): + if approve_handle not in open(approve_follows_filename, + encoding='utf-8').read(): try: - with open(approve_follows_filename, 'a+') as fp_approve: + with open(approve_follows_filename, 'a+', + encoding='utf-8') as fp_approve: fp_approve.write(approve_handle_stored + '\n') except OSError: print('EX: store_follow_request 2 ' + approve_follows_filename) @@ -706,7 +722,8 @@ def store_follow_request(base_dir: str, ' is already awaiting approval') else: try: - with open(approve_follows_filename, 'w+') as fp_approve: + with open(approve_follows_filename, 'w+', + encoding='utf-8') as fp_approve: fp_approve.write(approve_handle_stored + '\n') except OSError: print('EX: store_follow_request 3 ' + approve_follows_filename) @@ -907,10 +924,12 @@ def send_follow_request(session, base_dir: str, unfollowed_filename = \ acct_dir(base_dir, nickname, domain) + '/unfollowed.txt' if os.path.isfile(unfollowed_filename): - if follow_handle in open(unfollowed_filename).read(): + if follow_handle in open(unfollowed_filename, + encoding='utf-8').read(): unfollowed_file = None try: - with open(unfollowed_filename, 'r') as fp_unfoll: + with open(unfollowed_filename, 'r', + encoding='utf-8') as fp_unfoll: unfollowed_file = fp_unfoll.read() except OSError: print('EX: send_follow_request ' + unfollowed_filename) @@ -918,7 +937,8 @@ def send_follow_request(session, base_dir: str, unfollowed_file = \ unfollowed_file.replace(follow_handle + '\n', '') try: - with open(unfollowed_filename, 'w+') as fp_unfoll: + with open(unfollowed_filename, 'w+', + encoding='utf-8') as fp_unfoll: fp_unfoll.write(unfollowed_file) except OSError: print('EX: unable to write ' + unfollowed_filename) diff --git a/followingCalendar.py b/followingCalendar.py index 9c5f0340b..aae78b974 100644 --- a/followingCalendar.py +++ b/followingCalendar.py @@ -48,17 +48,20 @@ def receiving_calendar_events(base_dir: str, nickname: str, domain: str, # create a new calendar file from the following file following_handles = None try: - with open(following_filename, 'r') as following_file: + with open(following_filename, 'r', + encoding='utf-8') as following_file: following_handles = following_file.read() except OSError: print('EX: receiving_calendar_events ' + following_filename) if following_handles: try: - with open(calendar_filename, 'w+') as fp_cal: + with open(calendar_filename, 'w+', + encoding='utf-8') as fp_cal: fp_cal.write(following_handles) except OSError: print('EX: receiving_calendar_events 2 ' + calendar_filename) - return handle + '\n' in open(calendar_filename).read() + return handle + '\n' in open(calendar_filename, + encoding='utf-8').read() def _receive_calendar_events(base_dir: str, nickname: str, domain: str, @@ -79,7 +82,8 @@ def _receive_calendar_events(base_dir: str, nickname: str, domain: str, handle = following_nickname + '@' + following_domain # check that you are following this handle - if handle + '\n' not in open(following_filename).read(): + if handle + '\n' not in open(following_filename, + encoding='utf-8').read(): print('WARN: ' + handle + ' is not in ' + following_filename) return @@ -92,7 +96,8 @@ def _receive_calendar_events(base_dir: str, nickname: str, domain: str, if os.path.isfile(calendar_filename): print('Calendar file exists') try: - with open(calendar_filename, 'r') as calendar_file: + with open(calendar_filename, 'r', + encoding='utf-8') as calendar_file: following_handles = calendar_file.read() except OSError: print('EX: _receive_calendar_events ' + calendar_filename) @@ -101,13 +106,15 @@ def _receive_calendar_events(base_dir: str, nickname: str, domain: str, print('Creating calendar file ' + calendar_filename) following_handles = '' try: - with open(following_filename, 'r') as following_file: + with open(following_filename, 'r', + encoding='utf-8') as following_file: following_handles = following_file.read() except OSError: print('EX: _receive_calendar_events 2 ' + calendar_filename) if add: try: - with open(calendar_filename, 'w+') as fp_cal: + with open(calendar_filename, 'w+', + encoding='utf-8') as fp_cal: fp_cal.write(following_handles + handle + '\n') except OSError: print('EX: unable to write ' + calendar_filename) @@ -121,7 +128,8 @@ def _receive_calendar_events(base_dir: str, nickname: str, domain: str, # remove from calendar file following_handles = following_handles.replace(handle + '\n', '') try: - with open(calendar_filename, 'w+') as fp_cal: + with open(calendar_filename, 'w+', + encoding='utf-8') as fp_cal: fp_cal.write(following_handles) except OSError: print('EX: _receive_calendar_events 3 ' + calendar_filename) @@ -132,7 +140,8 @@ def _receive_calendar_events(base_dir: str, nickname: str, domain: str, # append to the list of handles following_handles += handle + '\n' try: - with open(calendar_filename, 'w+') as fp_cal: + with open(calendar_filename, 'w+', + encoding='utf-8') as fp_cal: fp_cal.write(following_handles) except OSError: print('EX: _receive_calendar_events 4 ' + calendar_filename) diff --git a/git.py b/git.py index f374fb3db..e0d51faa4 100644 --- a/git.py +++ b/git.py @@ -38,7 +38,7 @@ def _get_git_project_name(base_dir: str, nickname: str, domain: str, return None subject_line_words = subject.lower().split(' ') for word in subject_line_words: - if word in open(git_projects_filename).read(): + if word in open(git_projects_filename, encoding='utf-8').read(): return word return None @@ -208,11 +208,12 @@ def receive_git_patch(base_dir: str, nickname: str, domain: str, _git_add_from_handle(patch_str, '@' + from_nickname + '@' + from_domain) try: - with open(patch_filename, 'w+') as patch_file: + with open(patch_filename, 'w+', encoding='utf-8') as patch_file: patch_file.write(patch_str) patch_notify_filename = \ acct_dir(base_dir, nickname, domain) + '/.newPatchContent' - with open(patch_notify_filename, 'w+') as patch_file: + with open(patch_notify_filename, 'w+', + encoding='utf-8') as patch_file: patch_file.write(patch_str) return True except OSError as ex: diff --git a/happening.py b/happening.py index b5b0979d2..49df1a9d1 100644 --- a/happening.py +++ b/happening.py @@ -70,12 +70,15 @@ def _remove_event_from_timeline(event_id: str, tl_events_filename: str) -> None: """Removes the given event Id from the timeline """ - if event_id + '\n' not in open(tl_events_filename).read(): + if event_id + '\n' not in open(tl_events_filename, + encoding='utf-8').read(): return - with open(tl_events_filename, 'r') as fp_tl: + with open(tl_events_filename, 'r', + encoding='utf-8') as fp_tl: events_timeline = fp_tl.read().replace(event_id + '\n', '') try: - with open(tl_events_filename, 'w+') as fp2: + with open(tl_events_filename, 'w+', + encoding='utf-8') as fp2: fp2.write(events_timeline) except OSError: print('EX: ERROR: unable to save events timeline') @@ -135,7 +138,8 @@ def save_event_post(base_dir: str, handle: str, post_id: str, if os.path.isfile(tl_events_filename): _remove_event_from_timeline(event_id, tl_events_filename) try: - with open(tl_events_filename, 'r+') as tl_events_file: + with open(tl_events_filename, 'r+', + encoding='utf-8') as tl_events_file: content = tl_events_file.read() if event_id + '\n' not in content: tl_events_file.seek(0, 0) @@ -146,7 +150,8 @@ def save_event_post(base_dir: str, handle: str, post_id: str, return False else: try: - with open(tl_events_filename, 'w+') as tl_events_file: + with open(tl_events_filename, 'w+', + encoding='utf-8') as tl_events_file: tl_events_file.write(event_id + '\n') except OSError: print('EX: unable to write ' + tl_events_filename) @@ -161,13 +166,14 @@ def save_event_post(base_dir: str, handle: str, post_id: str, # Does this event post already exist within the calendar month? if os.path.isfile(calendar_filename): - if post_id in open(calendar_filename).read(): + if post_id in open(calendar_filename, + encoding='utf-8').read(): # Event post already exists return False # append the post Id to the file for the calendar month try: - with open(calendar_filename, 'a+') as calendar_file: + with open(calendar_filename, 'a+', encoding='utf-8') as calendar_file: calendar_file.write(post_id + '\n') except OSError: print('EX: unable to append ' + calendar_filename) @@ -179,7 +185,7 @@ def save_event_post(base_dir: str, handle: str, post_id: str, '/calendar?year=' + str(event_year) + '?month=' + \ str(event_month_number) + '?day=' + str(event_day_of_month) try: - with open(cal_notify_filename, 'w+') as cal_file: + with open(cal_notify_filename, 'w+', encoding='utf-8') as cal_file: cal_file.write(notify_str) except OSError: print('EX: unable to write ' + cal_notify_filename) @@ -255,7 +261,7 @@ def get_todays_events(base_dir: str, nickname: str, domain: str, calendar_post_ids = [] recreate_events_file = False - with open(calendar_filename, 'r') as events_file: + with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: post_id = post_id.replace('\n', '').replace('\r', '') post_filename = locate_post(base_dir, nickname, domain, post_id) @@ -321,7 +327,8 @@ def get_todays_events(base_dir: str, nickname: str, domain: str, # if some posts have been deleted then regenerate the calendar file if recreate_events_file: try: - with open(calendar_filename, 'w+') as calendar_file: + with open(calendar_filename, 'w+', + encoding='utf-8') as calendar_file: for post_id in calendar_post_ids: calendar_file.write(post_id + '\n') except OSError: @@ -553,7 +560,7 @@ def day_events_check(base_dir: str, nickname: str, domain: str, return False events_exist = False - with open(calendar_filename, 'r') as events_file: + with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: post_id = post_id.replace('\n', '').replace('\r', '') post_filename = locate_post(base_dir, nickname, domain, post_id) @@ -649,7 +656,8 @@ def get_this_weeks_events(base_dir: str, nickname: str, domain: str) -> {}: # if some posts have been deleted then regenerate the calendar file if recreate_events_file: try: - with open(calendar_filename, 'w+') as calendar_file: + with open(calendar_filename, 'w+', + encoding='utf-8') as calendar_file: for post_id in calendar_post_ids: calendar_file.write(post_id + '\n') except OSError: @@ -675,7 +683,7 @@ def get_calendar_events(base_dir: str, nickname: str, domain: str, calendar_post_ids = [] recreate_events_file = False - with open(calendar_filename, 'r') as events_file: + with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: post_id = post_id.replace('\n', '').replace('\r', '') post_filename = locate_post(base_dir, nickname, domain, post_id) @@ -730,7 +738,8 @@ def get_calendar_events(base_dir: str, nickname: str, domain: str, # if some posts have been deleted then regenerate the calendar file if recreate_events_file: try: - with open(calendar_filename, 'w+') as calendar_file: + with open(calendar_filename, 'w+', + encoding='utf-8') as calendar_file: for post_id in calendar_post_ids: calendar_file.write(post_id + '\n') except OSError: @@ -751,15 +760,15 @@ def remove_calendar_event(base_dir: str, nickname: str, domain: str, return if '/' in message_id: message_id = message_id.replace('/', '#') - if message_id not in open(calendar_filename).read(): + if message_id not in open(calendar_filename, encoding='utf-8').read(): return lines = None - with open(calendar_filename, 'r') as fp_cal: + with open(calendar_filename, 'r', encoding='utf-8') as fp_cal: lines = fp_cal.readlines() if not lines: return try: - with open(calendar_filename, 'w+') as fp_cal: + with open(calendar_filename, 'w+', encoding='utf-8') as fp_cal: for line in lines: if message_id not in line: fp_cal.write(line) diff --git a/inbox.py b/inbox.py index 1ed30a05f..07b53c5f2 100644 --- a/inbox.py +++ b/inbox.py @@ -245,7 +245,7 @@ def _store_last_post_id(base_dir: str, nickname: str, domain: str, os.mkdir(lastpost_dir) actor_filename = lastpost_dir + '/' + actor.replace('/', '#') try: - with open(actor_filename, 'w+') as fp_actor: + with open(actor_filename, 'w+', encoding='utf-8') as fp_actor: fp_actor.write(post_id) except OSError: print('EX: Unable to write last post id to ' + actor_filename) @@ -287,7 +287,8 @@ def _update_cached_hashtag_swarm(base_dir: str, nickname: str, domain: str, new_swarm_str = html_hash_tag_swarm(base_dir, actor, translate) if new_swarm_str: try: - with open(cached_hashtag_swarm_filename, 'w+') as fp_swarm: + with open(cached_hashtag_swarm_filename, 'w+', + encoding='utf-8') as fp_swarm: fp_swarm.write(new_swarm_str) return True except OSError: @@ -343,7 +344,7 @@ def store_hash_tags(base_dir: str, nickname: str, domain: str, hashtag_added = False if not os.path.isfile(tags_filename): try: - with open(tags_filename, 'w+') as tags_file: + with open(tags_filename, 'w+', encoding='utf-8') as tags_file: tags_file.write(tag_line) hashtag_added = True except OSError: @@ -351,14 +352,15 @@ def store_hash_tags(base_dir: str, nickname: str, domain: str, else: content = '' try: - with open(tags_filename, 'r') as tags_file: + with open(tags_filename, 'r', encoding='utf-8') as tags_file: content = tags_file.read() except OSError: pass if post_url not in content: content = tag_line + content try: - with open(tags_filename, 'w+') as tags_file: + with open(tags_filename, 'w+', + encoding='utf-8') as tags_file: tags_file.write(content) hashtag_added = True except OSError as ex: @@ -446,7 +448,7 @@ def valid_inbox(base_dir: str, nickname: str, domain: str) -> bool: if not os.path.isfile(filename): print('filename: ' + filename) return False - if 'postNickname' in open(filename).read(): + if 'postNickname' in open(filename, encoding='utf-8').read(): print('queue file incorrectly saved to ' + filename) return False break @@ -2056,7 +2058,6 @@ def _receive_undo_bookmark(recent_posts_cache: {}, undo_bookmarks_collection_entry(recent_posts_cache, base_dir, post_filename, - message_json['object']['url'], message_json['actor'], domain, debug) # regenerate the html bookmarked_post_json = load_json(post_filename, 0, 1) @@ -2384,7 +2385,8 @@ def _receive_announce(recent_posts_cache: {}, theme_name, system_language, 'inbox') try: - with open(post_filename + '.tts', 'w+') as ttsfile: + with open(post_filename + '.tts', 'w+', + encoding='utf-8') as ttsfile: ttsfile.write('\n') except OSError: print('EX: unable to write recent post ' + @@ -2551,18 +2553,22 @@ def populate_replies(base_dir: str, http_prefix: str, domain: str, post_replies_filename = post_filename.replace('.json', '.replies') message_id = remove_id_ending(message_json['id']) if os.path.isfile(post_replies_filename): - num_lines = sum(1 for line in open(post_replies_filename)) + num_lines = sum(1 for line in open(post_replies_filename, + encoding='utf-8')) if num_lines > max_replies: return False - if message_id not in open(post_replies_filename).read(): + if message_id not in open(post_replies_filename, + encoding='utf-8').read(): try: - with open(post_replies_filename, 'a+') as replies_file: + with open(post_replies_filename, 'a+', + encoding='utf-8') as replies_file: replies_file.write(message_id + '\n') except OSError: print('EX: unable to append ' + post_replies_filename) else: try: - with open(post_replies_filename, 'w+') as replies_file: + with open(post_replies_filename, 'w+', + encoding='utf-8') as replies_file: replies_file.write(message_id + '\n') except OSError: print('EX: unable to write ' + post_replies_filename) @@ -2775,7 +2781,7 @@ def _dm_notify(base_dir: str, handle: str, url: str) -> None: dm_file = account_dir + '/.newDM' if not os.path.isfile(dm_file): try: - with open(dm_file, 'w+') as fp_dm: + with open(dm_file, 'w+', encoding='utf-8') as fp_dm: fp_dm.write(url) except OSError: print('EX: unable to write ' + dm_file) @@ -2886,19 +2892,19 @@ def _like_notify(base_dir: str, domain: str, onion_domain: str, # was there a previous like notification? if os.path.isfile(prev_like_file): # is it the same as the current notification ? - with open(prev_like_file, 'r') as fp_like: + with open(prev_like_file, 'r', encoding='utf-8') as fp_like: prev_like_str = fp_like.read() if prev_like_str == like_str: return try: - with open(prev_like_file, 'w+') as fp_like: + with open(prev_like_file, 'w+', encoding='utf-8') as fp_like: fp_like.write(like_str) except OSError: print('EX: ERROR: unable to save previous like notification ' + prev_like_file) try: - with open(like_file, 'w+') as fp_like: + with open(like_file, 'w+', encoding='utf-8') as fp_like: fp_like.write(like_str) except OSError: print('EX: ERROR: unable to write like notification file ' + @@ -2931,7 +2937,7 @@ def _reaction_notify(base_dir: str, domain: str, onion_domain: str, reaction_file = account_dir + '/.newReaction' if os.path.isfile(reaction_file): - if '##sent##' not in open(reaction_file).read(): + if '##sent##' not in open(reaction_file, encoding='utf-8').read(): return reaction_nickname = get_nickname_from_actor(actor) @@ -2950,19 +2956,19 @@ def _reaction_notify(base_dir: str, domain: str, onion_domain: str, # was there a previous reaction notification? if os.path.isfile(prev_reaction_file): # is it the same as the current notification ? - with open(prev_reaction_file, 'r') as fp_react: + with open(prev_reaction_file, 'r', encoding='utf-8') as fp_react: prev_reaction_str = fp_react.read() if prev_reaction_str == reaction_str: return try: - with open(prev_reaction_file, 'w+') as fp_react: + with open(prev_reaction_file, 'w+', encoding='utf-8') as fp_react: fp_react.write(reaction_str) except OSError: print('EX: ERROR: unable to save previous reaction notification ' + prev_reaction_file) try: - with open(reaction_file, 'w+') as fp_react: + with open(reaction_file, 'w+', encoding='utf-8') as fp_react: fp_react.write(reaction_str) except OSError: print('EX: ERROR: unable to write reaction notification file ' + @@ -2980,12 +2986,12 @@ def _notify_post_arrival(base_dir: str, handle: str, url: str) -> None: notify_file = account_dir + '/.newNotifiedPost' if os.path.isfile(notify_file): # check that the same notification is not repeatedly sent - with open(notify_file, 'r') as fp_notify: + with open(notify_file, 'r', encoding='utf-8') as fp_notify: existing_notification_message = fp_notify.read() if url in existing_notification_message: return try: - with open(notify_file, 'w+') as fp_notify: + with open(notify_file, 'w+', encoding='utf-8') as fp_notify: fp_notify.write(url) except OSError: print('EX: unable to write ' + notify_file) @@ -3000,7 +3006,7 @@ def _reply_notify(base_dir: str, handle: str, url: str) -> None: reply_file = account_dir + '/.newReply' if not os.path.isfile(reply_file): try: - with open(reply_file, 'w+') as fp_reply: + with open(reply_file, 'w+', encoding='utf-8') as fp_reply: fp_reply.write(url) except OSError: print('EX: unable to write ' + reply_file) @@ -3018,7 +3024,7 @@ def _git_patch_notify(base_dir: str, handle: str, subject = subject.replace('[PATCH]', '').strip() handle = '@' + from_nickname + '@' + from_domain try: - with open(patch_file, 'w+') as fp_patch: + with open(patch_file, 'w+', encoding='utf-8') as fp_patch: fp_patch.write('git ' + handle + ' ' + subject) except OSError: print('EX: unable to write ' + patch_file) @@ -3175,7 +3181,7 @@ def inbox_update_index(boxname: str, base_dir: str, handle: str, written = False if os.path.isfile(index_filename): try: - with open(index_filename, 'r+') as index_file: + with open(index_filename, 'r+', encoding='utf-8') as index_file: content = index_file.read() if destination_filename + '\n' not in content: index_file.seek(0, 0) @@ -3186,7 +3192,7 @@ def inbox_update_index(boxname: str, base_dir: str, handle: str, print('EX: Failed to write entry to index ' + str(ex)) else: try: - with open(index_filename, 'w+') as index_file: + with open(index_filename, 'w+', encoding='utf-8') as index_file: index_file.write(destination_filename + '\n') written = True except OSError as ex: @@ -3218,13 +3224,15 @@ def _update_last_seen(base_dir: str, handle: str, actor: str) -> None: days_since_epoch = (curr_time - datetime.datetime(1970, 1, 1)).days # has the value changed? if os.path.isfile(last_seen_filename): - with open(last_seen_filename, 'r') as last_seen_file: + with open(last_seen_filename, 'r', + encoding='utf-8') as last_seen_file: days_since_epoch_file = last_seen_file.read() if int(days_since_epoch_file) == days_since_epoch: # value hasn't changed, so we can save writing anything to file return try: - with open(last_seen_filename, 'w+') as last_seen_file: + with open(last_seen_filename, 'w+', + encoding='utf-8') as last_seen_file: last_seen_file.write(str(days_since_epoch)) except OSError: print('EX: unable to write ' + last_seen_filename) @@ -4142,7 +4150,8 @@ def _inbox_after_initial(server, inbox_start_time, destination_filename_mitm = \ destination_filename.replace('.json', '') + '.mitm' try: - with open(destination_filename_mitm, 'w+') as mitm_file: + with open(destination_filename_mitm, 'w+', + encoding='utf-8') as mitm_file: mitm_file.write('\n') except OSError: print('EX: unable to write ' + destination_filename_mitm) @@ -4561,12 +4570,13 @@ def _check_json_signature(base_dir: str, queue_json: {}) -> (bool, bool): already_unknown = False if os.path.isfile(unknown_contexts_file): if unknown_context in \ - open(unknown_contexts_file).read(): + open(unknown_contexts_file, encoding='utf-8').read(): already_unknown = True if not already_unknown: try: - with open(unknown_contexts_file, 'a+') as unknown_file: + with open(unknown_contexts_file, 'a+', + encoding='utf-8') as unknown_file: unknown_file.write(unknown_context + '\n') except OSError: print('EX: unable to append ' + unknown_contexts_file) @@ -4579,7 +4589,7 @@ def _check_json_signature(base_dir: str, queue_json: {}) -> (bool, bool): already_unknown = False if os.path.isfile(unknown_signatures_file): if jwebsig_type in \ - open(unknown_signatures_file).read(): + open(unknown_signatures_file, encoding='utf-8').read(): already_unknown = True if not already_unknown: @@ -4818,7 +4828,8 @@ def _receive_follow_request(session, session_onion, session_i2p, print('Group cannot follow a group') return False try: - with open(followers_filename, 'r+') as followers_file: + with open(followers_filename, 'r+', + encoding='utf-8') as followers_file: content = followers_file.read() if approve_handle + '\n' not in content: followers_file.seek(0, 0) @@ -4834,7 +4845,8 @@ def _receive_follow_request(session, session_onion, session_i2p, str(ex)) else: try: - with open(followers_filename, 'w+') as followers_file: + with open(followers_filename, 'w+', + encoding='utf-8') as followers_file: followers_file.write(approve_handle + '\n') except OSError: print('EX: unable to write ' + followers_filename) @@ -5302,11 +5314,7 @@ def run_inbox_queue(server, if debug: print('DEBUG: No follow requests') - if receive_accept_reject(curr_session, - base_dir, http_prefix, domain, port, - send_threads, post_log, - cached_webfingers, person_cache, - queue_json['post'], + if receive_accept_reject(base_dir, domain, queue_json['post'], federation_list, debug, domain, onion_domain, i2p_domain): print('Queue: Accept/Reject received from ' + key_id) diff --git a/languages.py b/languages.py index 97918f737..7b4836692 100644 --- a/languages.py +++ b/languages.py @@ -39,7 +39,7 @@ def get_understood_languages(base_dir: str, http_prefix: str, """ person_url = local_actor_url(http_prefix, nickname, domain_full) actor_json = \ - get_person_from_cache(base_dir, person_url, person_cache, False) + get_person_from_cache(base_dir, person_url, person_cache) if not actor_json: print('WARN: unable to load actor to obtain languages ' + person_url) return [] @@ -117,7 +117,7 @@ def understood_post_language(base_dir: str, nickname: str, domain: str, return True person_url = local_actor_url(http_prefix, nickname, domain_full) actor_json = \ - get_person_from_cache(base_dir, person_url, person_cache, False) + get_person_from_cache(base_dir, person_url, person_cache) if not actor_json: print('WARN: unable to load actor to check languages ' + person_url) return False diff --git a/manualapprove.py b/manualapprove.py index 2a12f7c32..bda5a8f5e 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -38,7 +38,8 @@ def manual_deny_follow_request(session, session_onion, session_i2p, # has this handle already been rejected? rejected_follows_filename = accounts_dir + '/followrejects.txt' if os.path.isfile(rejected_follows_filename): - if deny_handle in open(rejected_follows_filename).read(): + if deny_handle in open(rejected_follows_filename, + encoding='utf-8').read(): remove_from_follow_requests(base_dir, nickname, domain, deny_handle, debug) print(deny_handle + @@ -49,7 +50,8 @@ def manual_deny_follow_request(session, session_onion, session_i2p, # Store rejected follows try: - with open(rejected_follows_filename, 'a+') as rejects_file: + with open(rejected_follows_filename, 'a+', + encoding='utf-8') as rejects_file: rejects_file.write(deny_handle + '\n') except OSError: print('EX: unable to append ' + rejected_follows_filename) @@ -153,7 +155,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p, # is the handle in the requests file? approve_follows_str = '' - with open(approve_follows_filename, 'r') as fp_foll: + with open(approve_follows_filename, 'r', encoding='utf-8') as fp_foll: approve_follows_str = fp_foll.read() exists = False approve_handle_full = approve_handle @@ -181,10 +183,12 @@ def manual_approve_follow_request(session, session_onion, session_i2p, '" ' + approve_follows_filename) return - with open(approve_follows_filename + '.new', 'w+') as approvefilenew: + with open(approve_follows_filename + '.new', 'w+', + encoding='utf-8') as approvefilenew: update_approved_followers = False follow_activity_filename = None - with open(approve_follows_filename, 'r') as approvefile: + with open(approve_follows_filename, 'r', + encoding='utf-8') as approvefile: for handle_of_follow_requester in approvefile: # is this the approved follow? if handle_of_follow_requester.startswith(approve_handle_full): @@ -276,9 +280,11 @@ def manual_approve_follow_request(session, session_onion, session_i2p, # update the followers print('Manual follow accept: updating ' + followers_filename) if os.path.isfile(followers_filename): - if approve_handle_full not in open(followers_filename).read(): + if approve_handle_full not in open(followers_filename, + encoding='utf-8').read(): try: - with open(followers_filename, 'r+') as followers_file: + with open(followers_filename, 'r+', + encoding='utf-8') as followers_file: content = followers_file.read() if approve_handle_full + '\n' not in content: followers_file.seek(0, 0) @@ -294,14 +300,16 @@ def manual_approve_follow_request(session, session_onion, session_i2p, print('Manual follow accept: first follower accepted for ' + handle + ' is ' + approve_handle_full) try: - with open(followers_filename, 'w+') as followers_file: + with open(followers_filename, 'w+', + encoding='utf-8') as followers_file: followers_file.write(approve_handle_full + '\n') except OSError: print('EX: unable to write ' + followers_filename) # only update the follow requests file if the follow is confirmed to be # in followers.txt - if approve_handle_full in open(followers_filename).read(): + if approve_handle_full in open(followers_filename, + encoding='utf-8').read(): # mark this handle as approved for following _approve_follower_handle(account_dir, approve_handle) # update the follow requests with the handles not yet approved diff --git a/media.py b/media.py index 046b824df..c87a34dcd 100644 --- a/media.py +++ b/media.py @@ -533,7 +533,7 @@ def _update_etag(media_filename: str) -> None: etag = sha1(data).hexdigest() # nosec # save the hash try: - with open(media_filename + '.etag', 'w+') as efile: + with open(media_filename + '.etag', 'w+', encoding='utf-8') as efile: efile.write(etag) except OSError: print('EX: _update_etag unable to write ' + diff --git a/metadata.py b/metadata.py index d3d488a12..7352582f9 100644 --- a/metadata.py +++ b/metadata.py @@ -105,7 +105,7 @@ def meta_data_instance(show_accounts: bool, rules_filename = \ base_dir + '/accounts/tos.md' if os.path.isfile(rules_filename): - with open(rules_filename, 'r') as fp_rules: + with open(rules_filename, 'r', encoding='utf-8') as fp_rules: rules_lines = fp_rules.readlines() rule_ctr = 1 for line in rules_lines: diff --git a/migrate.py b/migrate.py index 9ff4a68ff..d2252c45b 100644 --- a/migrate.py +++ b/migrate.py @@ -34,7 +34,7 @@ def _move_following_handles_for_account(base_dir: str, acct_dir(base_dir, nickname, domain) + '/following.txt' if not os.path.isfile(following_filename): return ctr - with open(following_filename, 'r') as fp_foll: + with open(following_filename, 'r', encoding='utf-8') as fp_foll: following_handles = fp_foll.readlines() for follow_handle in following_handles: follow_handle = follow_handle.strip("\n").strip("\r") @@ -129,7 +129,7 @@ def _update_moved_handle(base_dir: str, nickname: str, domain: str, following_filename = \ acct_dir(base_dir, nickname, domain) + '/following.txt' if os.path.isfile(following_filename): - with open(following_filename, 'r') as foll1: + with open(following_filename, 'r', encoding='utf-8') as foll1: following_handles = foll1.readlines() moved_to_handle = moved_to_nickname + '@' + moved_to_domain_full @@ -139,7 +139,7 @@ def _update_moved_handle(base_dir: str, nickname: str, domain: str, acct_dir(base_dir, nickname, domain) + '/refollow.txt' # unfollow the old handle - with open(following_filename, 'w+') as foll2: + with open(following_filename, 'w+', encoding='utf-8') as foll2: for follow_handle in following_handles: if follow_handle.strip("\n").strip("\r").lower() != \ handle_lower: @@ -157,22 +157,24 @@ def _update_moved_handle(base_dir: str, nickname: str, domain: str, # save the new handles to the refollow list if os.path.isfile(refollow_filename): - with open(refollow_filename, 'a+') as refoll: + with open(refollow_filename, 'a+', + encoding='utf-8') as refoll: refoll.write(moved_to_handle + '\n') else: - with open(refollow_filename, 'w+') as refoll: + with open(refollow_filename, 'w+', + encoding='utf-8') as refoll: refoll.write(moved_to_handle + '\n') followers_filename = \ acct_dir(base_dir, nickname, domain) + '/followers.txt' if os.path.isfile(followers_filename): - with open(followers_filename, 'r') as foll3: + with open(followers_filename, 'r', encoding='utf-8') as foll3: follower_handles = foll3.readlines() handle_lower = handle.lower() # remove followers who have moved - with open(followers_filename, 'w+') as foll4: + with open(followers_filename, 'w+', encoding='utf-8') as foll4: for follower_handle in follower_handles: if follower_handle.strip("\n").strip("\r").lower() != \ handle_lower: diff --git a/newsdaemon.py b/newsdaemon.py index 4c739925b..5019803b7 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -46,7 +46,7 @@ def _update_feeds_outbox_index(base_dir: str, domain: str, index_filename = base_path + '/outbox.index' if os.path.isfile(index_filename): - if post_id not in open(index_filename).read(): + if post_id not in open(index_filename, encoding='utf-8').read(): try: with open(index_filename, 'r+') as feeds_file: content = feeds_file.read() @@ -59,7 +59,7 @@ def _update_feeds_outbox_index(base_dir: str, domain: str, index_filename + ' ' + str(ex)) else: try: - with open(index_filename, 'w+') as feeds_file: + with open(index_filename, 'w+', encoding='utf-8') as feeds_file: feeds_file.write(post_id + '\n') except OSError: print('EX: unable to write ' + index_filename) @@ -70,7 +70,8 @@ def _save_arrived_time(base_dir: str, post_filename: str, """Saves the time when an rss post arrived to a file """ try: - with open(post_filename + '.arrived', 'w+') as arrived_file: + with open(post_filename + '.arrived', 'w+', + encoding='utf-8') as arrived_file: arrived_file.write(arrived) except OSError: print('EX: unable to write ' + post_filename + '.arrived') @@ -394,7 +395,7 @@ def _newswire_hashtag_processing(session, base_dir: str, post_json_object: {}, if not os.path.isfile(rules_filename): return True rules = [] - with open(rules_filename, 'r') as fp_rules: + with open(rules_filename, 'r', encoding='utf-8') as fp_rules: rules = fp_rules.readlines() domain_full = get_full_domain(domain, port) @@ -466,7 +467,7 @@ def _create_news_mirror(base_dir: str, domain: str, # no index for mirrors found return True removals = [] - with open(mirror_index_filename, 'r') as index_file: + with open(mirror_index_filename, 'r', encoding='utf-8') as index_file: # remove the oldest directories ctr = 0 while no_of_dirs > max_mirrored_articles: @@ -489,13 +490,15 @@ def _create_news_mirror(base_dir: str, domain: str, # remove the corresponding index entries if removals: index_content = '' - with open(mirror_index_filename, 'r') as index_file: + with open(mirror_index_filename, 'r', + encoding='utf-8') as index_file: index_content = index_file.read() for remove_post_id in removals: index_content = \ index_content.replace(remove_post_id + '\n', '') try: - with open(mirror_index_filename, 'w+') as index_file: + with open(mirror_index_filename, 'w+', + encoding='utf-8') as index_file: index_file.write(index_content) except OSError: print('EX: unable to write ' + mirror_index_filename) @@ -524,13 +527,15 @@ def _create_news_mirror(base_dir: str, domain: str, # append the post Id number to the index file if os.path.isfile(mirror_index_filename): try: - with open(mirror_index_filename, 'a+') as index_file: + with open(mirror_index_filename, 'a+', + encoding='utf-8') as index_file: index_file.write(post_id_number + '\n') except OSError: print('EX: unable to append ' + mirror_index_filename) else: try: - with open(mirror_index_filename, 'w+') as index_file: + with open(mirror_index_filename, 'w+', + encoding='utf-8') as index_file: index_file.write(post_id_number + '\n') except OSError: print('EX: unable to write ' + mirror_index_filename) diff --git a/newswire.py b/newswire.py index 8acd22364..f0360dcd9 100644 --- a/newswire.py +++ b/newswire.py @@ -116,7 +116,7 @@ def get_newswire_tags(text: str, max_tags: int) -> []: return tags -def limit_word_lengths(text: str, maxWordLength: int) -> str: +def limit_word_lengths(text: str, max_word_length: int) -> str: """Limits the maximum length of words so that the newswire column cannot become too wide """ @@ -125,8 +125,8 @@ def limit_word_lengths(text: str, maxWordLength: int) -> str: words = text.split(' ') result = '' for wrd in words: - if len(wrd) > maxWordLength: - wrd = wrd[:maxWordLength] + if len(wrd) > max_word_length: + wrd = wrd[:max_word_length] if result: result += ' ' result += wrd @@ -375,18 +375,18 @@ def load_hashtag_categories(base_dir: str, language: str) -> None: if not os.path.isfile(hashtag_categories_filename): return - with open(hashtag_categories_filename, 'r') as fp_cat: + with open(hashtag_categories_filename, 'r', encoding='utf-8') as fp_cat: xml_str = fp_cat.read() _xml2str_to_hashtag_categories(base_dir, xml_str, 1024, True) def _xml2str_to_hashtag_categories(base_dir: str, xml_str: str, - max_categories_feedItem_size_kb: int, + max_categories_feed_item_size_kb: int, force: bool = False) -> None: """Updates hashtag categories based upon an rss feed """ rss_items = xml_str.split('') - max_bytes = max_categories_feedItem_size_kb * 1024 + max_bytes = max_categories_feed_item_size_kb * 1024 for rss_item in rss_items: if not rss_item: continue @@ -488,7 +488,7 @@ def _valid_podcast_entry(base_dir: str, key: str, entry: {}) -> bool: post_url = entry['text'] if '://' not in post_url: return False - post_domain, post_port = get_domain_from_actor(post_url) + post_domain, _ = get_domain_from_actor(post_url) if not post_domain: return False if is_blocked_domain(base_dir, post_domain): @@ -720,7 +720,7 @@ def _xml2str_to_dict(base_dir: str, domain: str, xml_str: str, moderated: bool, mirrored: bool, max_posts_per_source: int, max_feed_item_size_kb: int, - max_categories_feedItem_size_kb: int, + max_categories_feed_item_size_kb: int, session, debug: bool, preferred_podcast_formats: []) -> {}: """Converts an xml RSS 2.0 string to a dictionary @@ -732,7 +732,7 @@ def _xml2str_to_dict(base_dir: str, domain: str, xml_str: str, # is this an rss feed containing hashtag categories? if '#categories' in xml_str: _xml2str_to_hashtag_categories(base_dir, xml_str, - max_categories_feedItem_size_kb) + max_categories_feed_item_size_kb) return {} rss_items = xml_str.split('') @@ -824,7 +824,7 @@ def _xml1str_to_dict(base_dir: str, domain: str, xml_str: str, moderated: bool, mirrored: bool, max_posts_per_source: int, max_feed_item_size_kb: int, - max_categories_feedItem_size_kb: int, + max_categories_feed_item_size_kb: int, session, debug: bool, preferred_podcast_formats: []) -> {}: """Converts an xml RSS 1.0 string to a dictionary @@ -838,7 +838,7 @@ def _xml1str_to_dict(base_dir: str, domain: str, xml_str: str, # is this an rss feed containing hashtag categories? if '#categories' in xml_str: _xml2str_to_hashtag_categories(base_dir, xml_str, - max_categories_feedItem_size_kb) + max_categories_feed_item_size_kb) return {} rss_items = xml_str.split(item_str) @@ -1226,7 +1226,7 @@ def _xml_str_to_dict(base_dir: str, domain: str, xml_str: str, moderated: bool, mirrored: bool, max_posts_per_source: int, max_feed_item_size_kb: int, - max_categories_feedItem_size_kb: int, + max_categories_feed_item_size_kb: int, session, debug: bool, preferred_podcast_formats: []) -> {}: """Converts an xml string to a dictionary @@ -1242,14 +1242,14 @@ def _xml_str_to_dict(base_dir: str, domain: str, xml_str: str, return _xml2str_to_dict(base_dir, domain, xml_str, moderated, mirrored, max_posts_per_source, max_feed_item_size_kb, - max_categories_feedItem_size_kb, + max_categories_feed_item_size_kb, session, debug, preferred_podcast_formats) if ' {}: """Returns an RSS url as a dict @@ -1318,7 +1318,7 @@ def get_rss(base_dir: str, domain: str, session, url: str, moderated, mirrored, max_posts_per_source, max_feed_item_size_kb, - max_categories_feedItem_size_kb, + max_categories_feed_item_size_kb, session, debug, preferred_podcast_formats) print('WARN: feed is too large, ' + @@ -1445,7 +1445,7 @@ def _add_account_blogs_to_newswire(base_dir: str, nickname: str, domain: str, if os.path.isfile(moderated_filename): moderated = True - with open(index_filename, 'r') as index_file: + with open(index_filename, 'r', encoding='utf-8') as index_file: post_filename = 'start' ctr = 0 while post_filename: @@ -1563,7 +1563,7 @@ def get_dict_from_newswire(session, base_dir: str, domain: str, max_posts_per_source: int, max_feed_size_kb: int, max_tags: int, max_feed_item_size_kb: int, max_newswire_posts: int, - max_categories_feedItem_size_kb: int, + max_categories_feed_item_size_kb: int, system_language: str, debug: bool, preferred_podcast_formats: [], timeout_sec: int) -> {}: @@ -1577,7 +1577,7 @@ def get_dict_from_newswire(session, base_dir: str, domain: str, # add rss feeds rss_feed = [] - with open(subscriptions_filename, 'r') as fp_sub: + with open(subscriptions_filename, 'r', encoding='utf-8') as fp_sub: rss_feed = fp_sub.readlines() result = {} for url in rss_feed: @@ -1607,7 +1607,7 @@ def get_dict_from_newswire(session, base_dir: str, domain: str, moderated, mirrored, max_posts_per_source, max_feed_size_kb, max_feed_item_size_kb, - max_categories_feedItem_size_kb, debug, + max_categories_feed_item_size_kb, debug, preferred_podcast_formats, timeout_sec) if items_list: diff --git a/notifyOnPost.py b/notifyOnPost.py index e82c68592..f25fa9b48 100644 --- a/notifyOnPost.py +++ b/notifyOnPost.py @@ -30,7 +30,8 @@ def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, handle = following_nickname + '@' + following_domain # check that you are following this handle - if handle + '\n' not in open(following_filename).read(): + if handle + '\n' not in open(following_filename, + encoding='utf-8').read(): print('WARN: ' + handle + ' is not in ' + following_filename) return @@ -42,16 +43,19 @@ def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, following_handles = '' if os.path.isfile(notify_on_post_filename): print('notify file exists') - with open(notify_on_post_filename, 'r') as calendar_file: + with open(notify_on_post_filename, 'r', + encoding='utf-8') as calendar_file: following_handles = calendar_file.read() else: # create a new notifyOnPost file from the following file print('Creating notifyOnPost file ' + notify_on_post_filename) following_handles = '' - with open(following_filename, 'r') as following_file: + with open(following_filename, 'r', + encoding='utf-8') as following_file: following_handles = following_file.read() if add: - with open(notify_on_post_filename, 'w+') as fp_notify: + with open(notify_on_post_filename, 'w+', + encoding='utf-8') as fp_notify: fp_notify.write(following_handles + handle + '\n') # already in the notifyOnPost file? @@ -62,7 +66,8 @@ def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, return # remove from calendar file following_handles = following_handles.replace(handle + '\n', '') - with open(notify_on_post_filename, 'w+') as fp_notify: + with open(notify_on_post_filename, 'w+', + encoding='utf-8') as fp_notify: fp_notify.write(following_handles) else: print(handle + ' not in notifyOnPost.txt') @@ -70,7 +75,8 @@ def _notify_on_post_arrival(base_dir: str, nickname: str, domain: str, if add: # append to the list of handles following_handles += handle + '\n' - with open(notify_on_post_filename, 'w+') as fp_notify: + with open(notify_on_post_filename, 'w+', + encoding='utf-8') as fp_notify: fp_notify.write(following_handles) @@ -104,6 +110,8 @@ def notify_when_person_posts(base_dir: str, nickname: str, domain: str, handle = following_nickname + '@' + following_domain if not os.path.isfile(notify_on_post_filename): # create a new notifyOnPost file - with open(notify_on_post_filename, 'w+') as fp_notify: + with open(notify_on_post_filename, 'w+', + encoding='utf-8') as fp_notify: fp_notify.write('') - return handle + '\n' in open(notify_on_post_filename).read() + return handle + '\n' in open(notify_on_post_filename, + encoding='utf-8').read() diff --git a/outbox.py b/outbox.py index 703bc9930..f1fe3aa68 100644 --- a/outbox.py +++ b/outbox.py @@ -601,8 +601,7 @@ def post_message_to_outbox(session, translate: {}, if debug: print('DEBUG: handle any undo announce requests') outbox_undo_announce(recent_posts_cache, - base_dir, http_prefix, - post_to_nickname, domain, port, + base_dir, post_to_nickname, domain, message_json, debug) if debug: @@ -628,9 +627,7 @@ def post_message_to_outbox(session, translate: {}, if debug: print('DEBUG: handle block requests') - outbox_block(base_dir, http_prefix, - post_to_nickname, domain, - port, + outbox_block(base_dir, post_to_nickname, domain, message_json, debug) if debug: diff --git a/person.py b/person.py index 26fe69b3f..13b08117d 100644 --- a/person.py +++ b/person.py @@ -354,7 +354,7 @@ def get_default_person_context() -> str: def _create_person_base(base_dir: str, nickname: str, domain: str, port: int, - http_prefix: str, saveToFile: bool, + http_prefix: str, save_to_file: bool, manual_follower_approval: bool, group_account: bool, password: str) -> (str, str, {}, {}): @@ -365,7 +365,7 @@ def _create_person_base(base_dir: str, nickname: str, domain: str, port: int, create_webfinger_endpoint(nickname, domain, port, http_prefix, public_key_pem, group_account) - if saveToFile: + if save_to_file: store_webfinger_endpoint(nickname, domain, port, base_dir, webfinger_endpoint) @@ -485,7 +485,7 @@ def _create_person_base(base_dir: str, nickname: str, domain: str, port: int, del new_person['following'] del new_person['attachment'] - if saveToFile: + if save_to_file: # save person to file people_subdir = '/accounts' if not os.path.isdir(base_dir + people_subdir): @@ -569,14 +569,14 @@ def register_account(base_dir: str, http_prefix: str, domain: str, port: int, def create_group(base_dir: str, nickname: str, domain: str, port: int, - http_prefix: str, saveToFile: bool, + http_prefix: str, save_to_file: bool, password: str = None) -> (str, str, {}, {}): """Returns a group """ (private_key_pem, public_key_pem, new_person, webfinger_endpoint) = create_person(base_dir, nickname, domain, port, - http_prefix, saveToFile, + http_prefix, save_to_file, False, password, True) return private_key_pem, public_key_pem, new_person, webfinger_endpoint @@ -621,7 +621,7 @@ def save_person_qrcode(base_dir: str, def create_person(base_dir: str, nickname: str, domain: str, port: int, - http_prefix: str, saveToFile: bool, + http_prefix: str, save_to_file: bool, manual_follower_approval: bool, password: str, group_account: bool = False) -> (str, str, {}, {}): @@ -650,7 +650,7 @@ def create_person(base_dir: str, nickname: str, domain: str, port: int, new_person, webfinger_endpoint) = _create_person_base(base_dir, nickname, domain, port, http_prefix, - saveToFile, + save_to_file, manual_follower, group_account, password) @@ -672,7 +672,7 @@ def create_person(base_dir: str, nickname: str, domain: str, port: int, follow_dms_filename = \ acct_dir(base_dir, nickname, domain) + '/.followDMs' try: - with open(follow_dms_filename, 'w+') as ffile: + with open(follow_dms_filename, 'w+', encoding='utf-8') as ffile: ffile.write('\n') except OSError: print('EX: unable to write ' + follow_dms_filename) @@ -682,7 +682,7 @@ def create_person(base_dir: str, nickname: str, domain: str, port: int, notify_likes_filename = \ acct_dir(base_dir, nickname, domain) + '/.notifyLikes' try: - with open(notify_likes_filename, 'w+') as nfile: + with open(notify_likes_filename, 'w+', encoding='utf-8') as nfile: nfile.write('\n') except OSError: print('EX: unable to write ' + notify_likes_filename) @@ -692,7 +692,8 @@ def create_person(base_dir: str, nickname: str, domain: str, port: int, notify_reactions_filename = \ acct_dir(base_dir, nickname, domain) + '/.notifyReactions' try: - with open(notify_reactions_filename, 'w+') as nfile: + with open(notify_reactions_filename, 'w+', + encoding='utf-8') as nfile: nfile.write('\n') except OSError: print('EX: unable to write ' + notify_reactions_filename) @@ -1050,8 +1051,8 @@ def person_box_json(recent_posts_cache: {}, def set_display_nickname(base_dir: str, nickname: str, domain: str, - displayName: str) -> bool: - if len(displayName) > 32: + display_name: str) -> bool: + if len(display_name) > 32: return False handle = nickname + '@' + domain filename = base_dir + '/accounts/' + handle + '.json' @@ -1061,7 +1062,7 @@ def set_display_nickname(base_dir: str, nickname: str, domain: str, person_json = load_json(filename) if not person_json: return False - person_json['name'] = displayName + person_json['name'] = display_name save_json(person_json, filename) return True @@ -1088,15 +1089,15 @@ def set_bio(base_dir: str, nickname: str, domain: str, bio: str) -> bool: def reenable_account(base_dir: str, nickname: str) -> None: - """Removes an account suspention + """Removes an account suspension """ suspended_filename = base_dir + '/accounts/suspended.txt' if os.path.isfile(suspended_filename): lines = [] - with open(suspended_filename, 'r') as fp_sus: + with open(suspended_filename, 'r', encoding='utf-8') as fp_sus: lines = fp_sus.readlines() try: - with open(suspended_filename, 'w+') as fp_sus: + with open(suspended_filename, 'w+', encoding='utf-8') as fp_sus: for suspended in lines: if suspended.strip('\n').strip('\r') != nickname: fp_sus.write(suspended) @@ -1118,7 +1119,7 @@ def suspend_account(base_dir: str, nickname: str, domain: str) -> None: # Don't suspend moderators moderators_file = base_dir + '/accounts/moderators.txt' if os.path.isfile(moderators_file): - with open(moderators_file, 'r') as fp_mod: + with open(moderators_file, 'r', encoding='utf-8') as fp_mod: lines = fp_mod.readlines() for moderator in lines: if moderator.strip('\n').strip('\r') == nickname: @@ -1139,19 +1140,19 @@ def suspend_account(base_dir: str, nickname: str, domain: str) -> None: suspended_filename = base_dir + '/accounts/suspended.txt' if os.path.isfile(suspended_filename): - with open(suspended_filename, 'r') as fp_sus: + with open(suspended_filename, 'r', encoding='utf-8') as fp_sus: lines = fp_sus.readlines() for suspended in lines: if suspended.strip('\n').strip('\r') == nickname: return try: - with open(suspended_filename, 'a+') as fp_sus: + with open(suspended_filename, 'a+', encoding='utf-8') as fp_sus: fp_sus.write(nickname + '\n') except OSError: print('EX: unable to append ' + suspended_filename) else: try: - with open(suspended_filename, 'w+') as fp_sus: + with open(suspended_filename, 'w+', encoding='utf-8') as fp_sus: fp_sus.write(nickname + '\n') except OSError: print('EX: unable to write ' + suspended_filename) @@ -1176,7 +1177,7 @@ def can_remove_post(base_dir: str, nickname: str, # is the post by a moderator? moderators_file = base_dir + '/accounts/moderators.txt' if os.path.isfile(moderators_file): - with open(moderators_file, 'r') as fp_mod: + with open(moderators_file, 'r', encoding='utf-8') as fp_mod: lines = fp_mod.readlines() for moderator in lines: if domain_full + '/users/' + \ @@ -1206,13 +1207,13 @@ def _remove_tags_for_nickname(base_dir: str, nickname: str, continue if not os.path.isfile(tag_filename): continue - if match_str not in open(tag_filename).read(): + if match_str not in open(tag_filename, encoding='utf-8').read(): continue lines = [] - with open(tag_filename, 'r') as fp_tag: + with open(tag_filename, 'r', encoding='utf-8') as fp_tag: lines = fp_tag.readlines() try: - with open(tag_filename, 'w+') as tag_file: + with open(tag_filename, 'w+', encoding='utf-8') as tag_file: for tagline in lines: if match_str not in tagline: tag_file.write(tagline) @@ -1234,7 +1235,7 @@ def remove_account(base_dir: str, nickname: str, # Don't remove moderators moderators_file = base_dir + '/accounts/moderators.txt' if os.path.isfile(moderators_file): - with open(moderators_file, 'r') as fp_mod: + with open(moderators_file, 'r', encoding='utf-8') as fp_mod: lines = fp_mod.readlines() for moderator in lines: if moderator.strip('\n') == nickname: @@ -1357,11 +1358,12 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str, snoozed_filename = acct_dir(base_dir, nickname, domain) + '/snoozed.txt' if not os.path.isfile(snoozed_filename): return False - if snooze_actor + ' ' not in open(snoozed_filename).read(): + if snooze_actor + ' ' not in open(snoozed_filename, + encoding='utf-8').read(): return False # remove the snooze entry if it has timed out replace_str = None - with open(snoozed_filename, 'r') as snoozed_file: + with open(snoozed_filename, 'r', encoding='utf-8') as snoozed_file: for line in snoozed_file: # is this the entry for the actor? if line.startswith(snooze_actor + ' '): @@ -1379,16 +1381,17 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str, break if replace_str: content = None - with open(snoozed_filename, 'r') as snoozed_file: + with open(snoozed_filename, 'r', encoding='utf-8') as snoozed_file: content = snoozed_file.read().replace(replace_str, '') if content: try: - with open(snoozed_filename, 'w+') as snoozfile: + with open(snoozed_filename, 'w+', + encoding='utf-8') as snoozfile: snoozfile.write(content) except OSError: print('EX: unable to write ' + snoozed_filename) - if snooze_actor + ' ' in open(snoozed_filename).read(): + if snooze_actor + ' ' in open(snoozed_filename, encoding='utf-8').read(): return True return False @@ -1403,10 +1406,11 @@ def person_snooze(base_dir: str, nickname: str, domain: str, return snoozed_filename = account_dir + '/snoozed.txt' if os.path.isfile(snoozed_filename): - if snooze_actor + ' ' in open(snoozed_filename).read(): + if snooze_actor + ' ' in open(snoozed_filename, + encoding='utf-8').read(): return try: - with open(snoozed_filename, 'a+') as snoozed_file: + with open(snoozed_filename, 'a+', encoding='utf-8') as snoozed_file: snoozed_file.write(snooze_actor + ' ' + str(int(time.time())) + '\n') except OSError: @@ -1424,21 +1428,23 @@ def person_unsnooze(base_dir: str, nickname: str, domain: str, snoozed_filename = account_dir + '/snoozed.txt' if not os.path.isfile(snoozed_filename): return - if snooze_actor + ' ' not in open(snoozed_filename).read(): + if snooze_actor + ' ' not in open(snoozed_filename, + encoding='utf-8').read(): return replace_str = None - with open(snoozed_filename, 'r') as snoozed_file: + with open(snoozed_filename, 'r', encoding='utf-8') as snoozed_file: for line in snoozed_file: if line.startswith(snooze_actor + ' '): replace_str = line break if replace_str: content = None - with open(snoozed_filename, 'r') as snoozed_file: + with open(snoozed_filename, 'r', encoding='utf-8') as snoozed_file: content = snoozed_file.read().replace(replace_str, '') if content: try: - with open(snoozed_filename, 'w+') as snoozfile: + with open(snoozed_filename, 'w+', + encoding='utf-8') as snoozfile: snoozfile.write(content) except OSError: print('EX: unable to write ' + snoozed_filename) @@ -1457,7 +1463,7 @@ def set_person_notes(base_dir: str, nickname: str, domain: str, os.mkdir(notes_dir) notes_filename = notes_dir + '/' + handle + '.txt' try: - with open(notes_filename, 'w+') as notes_file: + with open(notes_filename, 'w+', encoding='utf-8') as notes_file: notes_file.write(notes) except OSError: print('EX: unable to write ' + notes_filename) @@ -1680,8 +1686,7 @@ def get_person_avatar_url(base_dir: str, person_url: str, person_cache: {}, """Returns the avatar url for the person """ person_json = \ - get_person_from_cache(base_dir, person_url, person_cache, - allow_downloads) + get_person_from_cache(base_dir, person_url, person_cache) if not person_json: return None @@ -1703,7 +1708,7 @@ def get_person_avatar_url(base_dir: str, person_url: str, person_cache: {}, if ext != 'svg': return im_path content = '' - with open(im_filename, 'r') as fp_im: + with open(im_filename, 'r', encoding='utf-8') as fp_im: content = fp_im.read() if not dangerous_svg(content, False): return im_path diff --git a/petnames.py b/petnames.py index 39be1bd41..ddb970cd3 100644 --- a/petnames.py +++ b/petnames.py @@ -28,7 +28,7 @@ def set_pet_name(base_dir: str, nickname: str, domain: str, # does this entry already exist? if os.path.isfile(petnames_filename): - with open(petnames_filename, 'r') as petnames_file: + with open(petnames_filename, 'r', encoding='utf-8') as petnames_file: petnames_str = petnames_file.read() if entry in petnames_str: return True @@ -42,7 +42,8 @@ def set_pet_name(base_dir: str, nickname: str, domain: str, new_petnames_str += entry # save the updated petnames file try: - with open(petnames_filename, 'w+') as petnames_file: + with open(petnames_filename, 'w+', + encoding='utf-8') as petnames_file: petnames_file.write(new_petnames_str) except OSError: print('EX: unable to save ' + petnames_filename) @@ -50,7 +51,8 @@ def set_pet_name(base_dir: str, nickname: str, domain: str, return True # entry does not exist in the petnames file try: - with open(petnames_filename, 'a+') as petnames_file: + with open(petnames_filename, 'a+', + encoding='utf-8') as petnames_file: petnames_file.write(entry) except OSError: print('EX: unable to append ' + petnames_filename) @@ -59,7 +61,7 @@ def set_pet_name(base_dir: str, nickname: str, domain: str, # first entry try: - with open(petnames_filename, 'w+') as petnames_file: + with open(petnames_filename, 'w+', encoding='utf-8') as petnames_file: petnames_file.write(entry) except OSError: print('EX: unable to write ' + petnames_filename) @@ -79,7 +81,7 @@ def get_pet_name(base_dir: str, nickname: str, domain: str, if not os.path.isfile(petnames_filename): return '' - with open(petnames_filename, 'r') as petnames_file: + with open(petnames_filename, 'r', encoding='utf-8') as petnames_file: petnames_str = petnames_file.read() if ' ' + handle + '\n' in petnames_str: petnames_list = petnames_str.split('\n') @@ -106,7 +108,7 @@ def _get_pet_name_handle(base_dir: str, nickname: str, domain: str, if not os.path.isfile(petnames_filename): return '' - with open(petnames_filename, 'r') as petnames_file: + with open(petnames_filename, 'r', encoding='utf-8') as petnames_file: petnames_str = petnames_file.read() if petname + ' ' in petnames_str: petnames_list = petnames_str.split('\n') diff --git a/posts.py b/posts.py index 733a34c8b..5bb6577b0 100644 --- a/posts.py +++ b/posts.py @@ -110,7 +110,7 @@ def is_moderator(base_dir: str, nickname: str) -> bool: return True return False - with open(moderators_file, 'r') as fp_mod: + with open(moderators_file, 'r', encoding='utf-8') as fp_mod: lines = fp_mod.readlines() if len(lines) == 0: admin_name = get_config_param(base_dir, 'admin') @@ -134,7 +134,7 @@ def no_of_followers_on_domain(base_dir: str, handle: str, return 0 ctr = 0 - with open(filename, 'r') as followers_file: + with open(filename, 'r', encoding='utf-8') as followers_file: for follower_handle in followers_file: if '@' in follower_handle: follower_domain = follower_handle.split('@')[1] @@ -154,7 +154,7 @@ def _get_local_private_key(base_dir: str, nickname: str, domain: str) -> str: key_filename = base_dir + '/keys/private/' + handle.lower() + '.key' if not os.path.isfile(key_filename): return None - with open(key_filename, 'r') as pem_file: + with open(key_filename, 'r', encoding='utf-8') as pem_file: return pem_file.read() return None @@ -175,7 +175,7 @@ def _get_local_public_key(base_dir: str, nickname: str, domain: str) -> str: key_filename = base_dir + '/keys/public/' + handle.lower() + '.key' if not os.path.isfile(key_filename): return None - with open(key_filename, 'r') as pem_file: + with open(key_filename, 'r', encoding='utf-8') as pem_file: return pem_file.read() return None @@ -313,7 +313,7 @@ def _get_person_box_actor(session, base_dir: str, actor: str, """Returns the actor json for the given actor url """ person_json = \ - get_person_from_cache(base_dir, actor, person_cache, True) + get_person_from_cache(base_dir, actor, person_cache) if person_json: return person_json @@ -961,16 +961,16 @@ def _update_hashtags_index(base_dir: str, tag: {}, new_post_id: str) -> None: if not os.path.isfile(tags_filename): # create a new tags index file try: - with open(tags_filename, 'w+') as tags_file: + with open(tags_filename, 'w+', encoding='utf-8') as tags_file: tags_file.write(tagline) except OSError: print('EX: _update_hashtags_index unable to write tags file ' + tags_filename) else: # prepend to tags index file - if tagline not in open(tags_filename).read(): + if tagline not in open(tags_filename, encoding='utf-8').read(): try: - with open(tags_filename, 'r+') as tags_file: + with open(tags_filename, 'r+', encoding='utf-8') as tags_file: content = tags_file.read() if tagline not in content: tags_file.seek(0, 0) @@ -990,9 +990,11 @@ def _add_schedule_post(base_dir: str, nickname: str, domain: str, index_str = event_date_str + ' ' + post_id.replace('/', '#') if os.path.isfile(schedule_index_filename): - if index_str not in open(schedule_index_filename).read(): + if index_str not in open(schedule_index_filename, + encoding='utf-8').read(): try: - with open(schedule_index_filename, 'r+') as schedule_file: + with open(schedule_index_filename, 'r+', + encoding='utf-8') as schedule_file: content = schedule_file.read() if index_str + '\n' not in content: schedule_file.seek(0, 0) @@ -1003,7 +1005,8 @@ def _add_schedule_post(base_dir: str, nickname: str, domain: str, schedule_index_filename + ' ' + str(ex)) else: try: - with open(schedule_index_filename, 'w+') as schedule_file: + with open(schedule_index_filename, 'w+', + encoding='utf-8') as schedule_file: schedule_file.write(index_str + '\n') except OSError as ex: print('EX: Failed to write entry to scheduled posts index2 ' + @@ -1029,7 +1032,7 @@ def _load_auto_cw(base_dir: str, nickname: str, domain: str) -> []: if not os.path.isfile(filename): return [] try: - with open(filename, 'r') as fp_auto: + with open(filename, 'r', encoding='utf-8') as fp_auto: return fp_auto.readlines() except OSError: print('EX: unable to load auto cw file ' + filename) @@ -1371,7 +1374,7 @@ def _create_post_mod_report(base_dir: str, # save to index file moderation_index_file = base_dir + '/accounts/moderation.txt' try: - with open(moderation_index_file, 'a+') as mod_file: + with open(moderation_index_file, 'a+', encoding='utf-8') as mod_file: mod_file.write(new_post_id + '\n') except OSError: print('EX: unable to write moderation index file ' + @@ -1707,7 +1710,7 @@ def pin_post(base_dir: str, nickname: str, domain: str, account_dir = acct_dir(base_dir, nickname, domain) pinned_filename = account_dir + '/pinToProfile.txt' try: - with open(pinned_filename, 'w+') as pin_file: + with open(pinned_filename, 'w+', encoding='utf-8') as pin_file: pin_file.write(pinned_content) except OSError: print('EX: unable to write ' + pinned_filename) @@ -1737,7 +1740,7 @@ def get_pinned_post_as_json(base_dir: str, http_prefix: str, actor = local_actor_url(http_prefix, nickname, domain_full) if os.path.isfile(pinned_filename): pinned_content = None - with open(pinned_filename, 'r') as pin_file: + with open(pinned_filename, 'r', encoding='utf-8') as pin_file: pinned_content = pin_file.read() if pinned_content: pinned_post_json = { @@ -1816,7 +1819,7 @@ def regenerate_index_for_box(base_dir: str, result = '' try: - with open(box_index_filename, 'w+') as fp_box: + with open(box_index_filename, 'w+', encoding='utf-8') as fp_box: for line in index_lines: result += line + '\n' fp_box.write(line + '\n') @@ -1886,7 +1889,7 @@ def _append_citations_to_blog_post(base_dir: str, if not os.path.isfile(citations_filename): return citations_separator = '#####' - with open(citations_filename, 'r') as fp_cit: + with open(citations_filename, 'r', encoding='utf-8') as fp_cit: citations = fp_cit.readlines() for line in citations: if citations_separator not in line: @@ -2212,7 +2215,7 @@ def create_report_post(base_dir: str, moderators_list = [] moderators_file = base_dir + '/accounts/moderators.txt' if os.path.isfile(moderators_file): - with open(moderators_file, 'r') as fp_mod: + with open(moderators_file, 'r', encoding='utf-8') as fp_mod: for line in fp_mod: line = line.strip('\n').strip('\r') if line.startswith('#'): @@ -2284,7 +2287,7 @@ def create_report_post(base_dir: str, if os.path.isfile(new_report_file): continue try: - with open(new_report_file, 'w+') as fp_report: + with open(new_report_file, 'w+', encoding='utf-8') as fp_report: fp_report.write(to_url + '/moderation') except OSError: print('EX: create_report_post unable to write ' + new_report_file) @@ -2362,10 +2365,12 @@ def thread_send_post(session, post_json_str: str, federation_list: [], # save the log file post_log_filename = base_dir + '/post.log' if os.path.isfile(post_log_filename): - with open(post_log_filename, 'a+') as log_file: + with open(post_log_filename, 'a+', + encoding='utf-8') as log_file: log_file.write(log_str + '\n') else: - with open(post_log_filename, 'w+') as log_file: + with open(post_log_filename, 'w+', + encoding='utf-8') as log_file: log_file.write(log_str + '\n') if post_result: @@ -2702,7 +2707,7 @@ def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}: if not os.path.isfile(followers_filename): return None grouped = {} - with open(followers_filename, 'r') as foll_file: + with open(followers_filename, 'r', encoding='utf-8') as foll_file: for follower_handle in foll_file: if '@' not in follower_handle: continue @@ -3701,7 +3706,8 @@ def create_moderation(base_dir: str, nickname: str, domain: str, port: int, if is_moderator(base_dir, nickname): moderation_index_file = base_dir + '/accounts/moderation.txt' if os.path.isfile(moderation_index_file): - with open(moderation_index_file, 'r') as index_file: + with open(moderation_index_file, 'r', + encoding='utf-8') as index_file: lines = index_file.readlines() box_header['totalItems'] = len(lines) if header_only: @@ -3831,7 +3837,7 @@ def _add_post_to_timeline(file_path: str, boxname: str, posts_in_box: [], box_actor: str) -> bool: """ Reads a post from file and decides whether it is valid """ - with open(file_path, 'r') as post_file: + with open(file_path, 'r', encoding='utf-8') as post_file: post_str = post_file.read() if file_path.endswith('.json'): @@ -4011,7 +4017,7 @@ def _create_box_indexed(recent_posts_cache: {}, total_posts_count = 0 posts_added_to_timeline = 0 if os.path.isfile(index_filename): - with open(index_filename, 'r') as index_file: + with open(index_filename, 'r', encoding='utf-8') as index_file: posts_added_to_timeline = 0 while posts_added_to_timeline < items_per_page: post_filename = index_file.readline() @@ -4267,7 +4273,7 @@ def archive_posts_for_person(http_prefix: str, nickname: str, domain: str, index_ctr = 0 # get the existing index entries as a string new_index = '' - with open(index_filename, 'r') as index_file: + with open(index_filename, 'r', encoding='utf-8') as index_file: for post_id in index_file: new_index += post_id index_ctr += 1 @@ -4275,7 +4281,7 @@ def archive_posts_for_person(http_prefix: str, nickname: str, domain: str, break # save the new index file if len(new_index) > 0: - with open(index_filename, 'w+') as index_file: + with open(index_filename, 'w+', encoding='utf-8') as index_file: index_file.write(new_index) posts_in_box_dict = {} @@ -4288,7 +4294,7 @@ def archive_posts_for_person(http_prefix: str, nickname: str, domain: str, # Time of file creation full_filename = os.path.join(box_dir, post_filename) if os.path.isfile(full_filename): - content = open(full_filename).read() + content = open(full_filename, encoding='utf-8').read() if '"published":' in content: published_str = content.split('"published":')[1] if '"' in published_str: @@ -4617,7 +4623,7 @@ def get_public_post_domains_blocked(session, base_dir: str, # read the blocked domains as a single string blocked_str = '' - with open(blocking_filename, 'r') as fp_block: + with open(blocking_filename, 'r', encoding='utf-8') as fp_block: blocked_str = fp_block.read() blocked_domains = [] @@ -4669,7 +4675,8 @@ def check_domains(session, base_dir: str, update_follower_warnings = False follower_warning_str = '' if os.path.isfile(follower_warning_filename): - with open(follower_warning_filename, 'r') as fp_warn: + with open(follower_warning_filename, 'r', + encoding='utf-8') as fp_warn: follower_warning_str = fp_warn.read() if single_check: @@ -4719,7 +4726,8 @@ def check_domains(session, base_dir: str, update_follower_warnings = True if update_follower_warnings and follower_warning_str: - with open(follower_warning_filename, 'w+') as fp_warn: + with open(follower_warning_filename, 'w+', + encoding='utf-8') as fp_warn: fp_warn.write(follower_warning_str) if not single_check: print(follower_warning_str) @@ -4731,7 +4739,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str, pub_str = 'https://www.w3.org/ns/activitystreams#Public' # populate the items list with replies replies_boxes = ('outbox', 'inbox') - with open(post_replies_filename, 'r') as replies_file: + with open(post_replies_filename, 'r', encoding='utf-8') as replies_file: for message_id in replies_file: reply_found = False # examine inbox and outbox @@ -4743,7 +4751,8 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str, message_id2.replace('/', '#') + '.json' if os.path.isfile(search_filename): if authorized or \ - pub_str in open(search_filename).read(): + pub_str in open(search_filename, + encoding='utf-8').read(): post_json_object = load_json(search_filename) if post_json_object: if post_json_object['object'].get('cc'): @@ -4770,7 +4779,8 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str, message_id2.replace('/', '#') + '.json' if os.path.isfile(search_filename): if authorized or \ - pub_str in open(search_filename).read(): + pub_str in open(search_filename, + encoding='utf-8').read(): # get the json of the reply and append it to # the collection post_json_object = load_json(search_filename) @@ -4799,7 +4809,8 @@ def _reject_announce(announce_filename: str, # reject the post referenced by the announce activity object if not os.path.isfile(announce_filename + '.reject'): - with open(announce_filename + '.reject', 'w+') as reject_announce_file: + with open(announce_filename + '.reject', 'w+', + encoding='utf-8') as reject_announce_file: reject_announce_file.write('\n') @@ -5565,7 +5576,8 @@ def edited_post_filename(base_dir: str, nickname: str, domain: str, post_id = remove_id_ending(post_json_object['object']['id']) lastpost_id = None try: - with open(actor_filename, 'r') as fp_actor: + with open(actor_filename, 'r', + encoding='utf-8') as fp_actor: lastpost_id = fp_actor.read() except OSError: print('EX: edited_post_filename unable to read ' + actor_filename) diff --git a/question.py b/question.py index 9514300a1..5ef6829f3 100644 --- a/question.py +++ b/question.py @@ -66,17 +66,20 @@ def question_update_votes(base_dir: str, nickname: str, domain: str, if not os.path.isfile(voters_filename): # create a new voters file try: - with open(voters_filename, 'w+') as voters_file: + with open(voters_filename, 'w+', + encoding='utf-8') as voters_file: voters_file.write(reply_json['actor'] + voters_file_separator + found_answer + '\n') except OSError: print('EX: unable to write voters file ' + voters_filename) else: - if reply_json['actor'] not in open(voters_filename).read(): + if reply_json['actor'] not in open(voters_filename, + encoding='utf-8').read(): # append to the voters file try: - with open(voters_filename, 'a+') as voters_file: + with open(voters_filename, 'a+', + encoding='utf-8') as voters_file: voters_file.write(reply_json['actor'] + voters_file_separator + found_answer + '\n') @@ -84,7 +87,8 @@ def question_update_votes(base_dir: str, nickname: str, domain: str, print('EX: unable to append to voters file ' + voters_filename) else: # change an entry in the voters file - with open(voters_filename, 'r') as voters_file: + with open(voters_filename, 'r', + encoding='utf-8') as voters_file: lines = voters_file.readlines() newlines = [] save_voters_file = False @@ -101,7 +105,8 @@ def question_update_votes(base_dir: str, nickname: str, domain: str, newlines.append(vote_line) if save_voters_file: try: - with open(voters_filename, 'w+') as voters_file: + with open(voters_filename, 'w+', + encoding='utf-8') as voters_file: for vote_line in newlines: voters_file.write(vote_line) except OSError: @@ -115,7 +120,7 @@ def question_update_votes(base_dir: str, nickname: str, domain: str, if not possible_answer.get('name'): continue total_items = 0 - with open(voters_filename, 'r') as voters_file: + with open(voters_filename, 'r', encoding='utf-8') as voters_file: lines = voters_file.readlines() for vote_line in lines: if vote_line.endswith(voters_file_separator + diff --git a/reaction.py b/reaction.py index ef9f11627..2c13fe589 100644 --- a/reaction.py +++ b/reaction.py @@ -452,7 +452,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None: common_reactions = None if os.path.isfile(common_reactions_filename): try: - with open(common_reactions_filename, 'r') as fp_react: + with open(common_reactions_filename, 'r', + encoding='utf-8') as fp_react: common_reactions = fp_react.readlines() except OSError: print('EX: unable to load common reactions file') @@ -476,7 +477,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None: new_common_reactions.append(str(1).zfill(16) + ' ' + emoji_content) new_common_reactions.sort(reverse=True) try: - with open(common_reactions_filename, 'w+') as fp_react: + with open(common_reactions_filename, 'w+', + encoding='utf-8') as fp_react: for line in new_common_reactions: fp_react.write(line + '\n') except OSError: @@ -485,7 +487,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None: else: line = str(1).zfill(16) + ' ' + emoji_content + '\n' try: - with open(common_reactions_filename, 'w+') as fp_react: + with open(common_reactions_filename, 'w+', + encoding='utf-8') as fp_react: fp_react.write(line) except OSError: print('EX: error writing common reactions 2') diff --git a/roles.py b/roles.py index 293990889..7fc649d79 100644 --- a/roles.py +++ b/roles.py @@ -28,7 +28,8 @@ def _clear_role_status(base_dir: str, role: str) -> None: if not filename.endswith(".json"): continue filename = os.path.join(base_dir + '/accounts/', filename) - if '"' + role + '"' not in open(filename).read(): + if '"' + role + '"' not in open(filename, + encoding='utf-8').read(): continue actor_json = load_json(filename) if not actor_json: @@ -84,7 +85,7 @@ def _add_role(base_dir: str, nickname: str, domain: str, lines = [] try: - with open(role_file, 'r') as fp_role: + with open(role_file, 'r', encoding='utf-8') as fp_role: lines = fp_role.readlines() except OSError: print('EX: _add_role, failed to read roles file ' + role_file) @@ -96,7 +97,7 @@ def _add_role(base_dir: str, nickname: str, domain: str, lines.append(nickname) try: - with open(role_file, 'w+') as fp_role: + with open(role_file, 'w+', encoding='utf-8') as fp_role: for role_nickname in lines: role_nickname = role_nickname.strip('\n').strip('\r') if len(role_nickname) < 2: @@ -108,7 +109,7 @@ def _add_role(base_dir: str, nickname: str, domain: str, print('EX: _add_role, failed to write roles file1 ' + role_file) else: try: - with open(role_file, 'w+') as fp_role: + with open(role_file, 'w+', encoding='utf-8') as fp_role: account_dir = acct_dir(base_dir, nickname, domain) if os.path.isdir(account_dir): fp_role.write(nickname + '\n') @@ -125,13 +126,13 @@ def _remove_role(base_dir: str, nickname: str, role_filename: str) -> None: return try: - with open(role_file, 'r') as fp_role: + with open(role_file, 'r', encoding='utf-8') as fp_role: lines = fp_role.readlines() except OSError: print('EX: _remove_role, failed to read roles file ' + role_file) try: - with open(role_file, 'w+') as fp_role: + with open(role_file, 'w+', encoding='utf-8') as fp_role: for role_nickname in lines: role_nickname = role_nickname.strip('\n').strip('\r') if len(role_nickname) > 1 and role_nickname != nickname: diff --git a/schedule.py b/schedule.py index 025a0f02e..810474b08 100644 --- a/schedule.py +++ b/schedule.py @@ -36,7 +36,7 @@ def _update_post_schedule(base_dir: str, handle: str, httpd, index_lines = [] delete_schedule_post = False nickname = handle.split('@')[0] - with open(schedule_index_filename, 'r') as sched_file: + with open(schedule_index_filename, 'r', encoding='utf-8') as sched_file: for line in sched_file: if ' ' not in line: continue @@ -163,7 +163,7 @@ def _update_post_schedule(base_dir: str, handle: str, httpd, # write the new schedule index file schedule_index_file = \ base_dir + '/accounts/' + handle + '/schedule.index' - with open(schedule_index_file, 'w+') as schedule_file: + with open(schedule_index_file, 'w+', encoding='utf-8') as schedule_file: for line in index_lines: schedule_file.write(line) diff --git a/shares.py b/shares.py index 202573838..548838958 100644 --- a/shares.py +++ b/shares.py @@ -291,7 +291,7 @@ def _indicate_new_share_available(base_dir: str, http_prefix: str, local_actor = \ local_actor_url(http_prefix, account_nickname, domain_full) try: - with open(new_share_file, 'w+') as fp_new: + with open(new_share_file, 'w+', encoding='utf-8') as fp_new: if shares_file_type == 'shares': fp_new.write(local_actor + '/tlshares') else: @@ -1639,7 +1639,7 @@ def _generate_next_shares_token_update(base_dir: str, token_update_filename = token_update_dir + '/.tokenUpdate' next_update_sec = None if os.path.isfile(token_update_filename): - with open(token_update_filename, 'r') as fp_tok: + with open(token_update_filename, 'r', encoding='utf-8') as fp_tok: next_update_str = fp_tok.read() if next_update_str: if next_update_str.isdigit(): @@ -1658,7 +1658,7 @@ def _generate_next_shares_token_update(base_dir: str, next_update_sec = curr_time + next_update_interval updated = True if updated: - with open(token_update_filename, 'w+') as fp_tok: + with open(token_update_filename, 'w+', encoding='utf-8') as fp_tok: fp_tok.write(str(next_update_sec)) @@ -1683,7 +1683,7 @@ def _regenerate_shares_token(base_dir: str, domain_full: str, if not os.path.isfile(token_update_filename): return next_update_sec = None - with open(token_update_filename, 'r') as fp_tok: + with open(token_update_filename, 'r', encoding='utf-8') as fp_tok: next_update_str = fp_tok.read() if next_update_str: if next_update_str.isdigit(): diff --git a/speaker.py b/speaker.py index e0e538a89..1cb5fcb7c 100644 --- a/speaker.py +++ b/speaker.py @@ -146,7 +146,7 @@ def _speaker_pronounce(base_dir: str, say_text: str, translate: {}) -> str: ")": "," } if os.path.isfile(pronounce_filename): - with open(pronounce_filename, 'r') as fp_pro: + with open(pronounce_filename, 'r', encoding='utf-8') as fp_pro: pronounce_list = fp_pro.readlines() for conversion in pronounce_list: separator = None @@ -507,7 +507,7 @@ def _post_to_speaker_json(base_dir: str, http_prefix: str, accounts_dir = acct_dir(base_dir, nickname, domain_full) approve_follows_filename = accounts_dir + '/followrequests.txt' if os.path.isfile(approve_follows_filename): - with open(approve_follows_filename, 'r') as fp_foll: + with open(approve_follows_filename, 'r', encoding='utf-8') as fp_foll: follows = fp_foll.readlines() if len(follows) > 0: follow_requests_exist = True @@ -525,7 +525,7 @@ def _post_to_speaker_json(base_dir: str, http_prefix: str, liked_by = '' like_filename = accounts_dir + '/.newLike' if os.path.isfile(like_filename): - with open(like_filename, 'r') as fp_like: + with open(like_filename, 'r', encoding='utf-8') as fp_like: liked_by = fp_like.read() calendar_filename = accounts_dir + '/.newCalendar' post_cal = os.path.isfile(calendar_filename) @@ -586,7 +586,7 @@ def update_speaker(base_dir: str, http_prefix: str, system_language, gender, box_name) try: - with open(cached_ssml_filename, 'w+') as fp_ssml: + with open(cached_ssml_filename, 'w+', encoding='utf-8') as fp_ssml: fp_ssml.write(ssml_str) except OSError: print('EX: unable to write ssml ' + cached_ssml_filename) diff --git a/tests.py b/tests.py index 07a8c3de1..8168c2c33 100644 --- a/tests.py +++ b/tests.py @@ -679,7 +679,7 @@ def _test_cache(): } person_cache = {} store_person_in_cache(None, person_url, person_json, person_cache, True) - result = get_person_from_cache(None, person_url, person_cache, True) + result = get_person_from_cache(None, person_url, person_cache) assert result['id'] == 123456 assert result['test'] == 'This is a test' @@ -705,7 +705,7 @@ def _test_threads(): def create_server_alice(path: str, domain: str, port: int, bob_address: str, federation_list: [], - hasFollows: bool, hasPosts: bool, + has_follows: bool, has_posts: bool, send_threads: []): print('Creating test server: Alice on port ' + str(port)) if os.path.isdir(path): @@ -735,12 +735,12 @@ def create_server_alice(path: str, domain: str, port: int, delete_all_posts(path, nickname, domain, 'outbox') assert set_skill_level(path, nickname, domain, 'hacking', 90) assert set_role(path, nickname, domain, 'guru') - if hasFollows: + if has_follows: follow_person(path, nickname, domain, 'bob', bob_address, federation_list, False, False) add_follower_of_person(path, nickname, domain, 'bob', bob_address, federation_list, False, False) - if hasPosts: + if has_posts: test_save_to_file = True client_to_server = False test_comments_enabled = True @@ -865,7 +865,7 @@ def create_server_alice(path: str, domain: str, port: int, def create_server_bob(path: str, domain: str, port: int, alice_address: str, federation_list: [], - hasFollows: bool, hasPosts: bool, + has_follows: bool, has_posts: bool, send_threads: []): print('Creating test server: Bob on port ' + str(port)) if os.path.isdir(path): @@ -894,13 +894,13 @@ def create_server_bob(path: str, domain: str, port: int, assert wf_endpoint delete_all_posts(path, nickname, domain, 'inbox') delete_all_posts(path, nickname, domain, 'outbox') - if hasFollows and alice_address: + if has_follows and alice_address: follow_person(path, nickname, domain, 'alice', alice_address, federation_list, False, False) add_follower_of_person(path, nickname, domain, 'alice', alice_address, federation_list, False, False) - if hasPosts: + if has_posts: test_save_to_file = True test_comments_enabled = True test_attach_image_filename = None @@ -1023,7 +1023,7 @@ def create_server_bob(path: str, domain: str, port: int, def create_server_eve(path: str, domain: str, port: int, federation_list: [], - hasFollows: bool, hasPosts: bool, + has_follows: bool, has_posts: bool, send_threads: []): print('Creating test server: Eve on port ' + str(port)) if os.path.isdir(path): @@ -1104,7 +1104,7 @@ def create_server_eve(path: str, domain: str, port: int, federation_list: [], def create_server_group(path: str, domain: str, port: int, federation_list: [], - hasFollows: bool, hasPosts: bool, + has_follows: bool, has_posts: bool, send_threads: []): print('Creating test server: Group on port ' + str(port)) if os.path.isdir(path): @@ -1418,7 +1418,7 @@ def test_post_message_between_servers(base_dir: str) -> None: bob_domain, None, None) for _ in range(20): - if 'likes' in open(outbox_post_filename).read(): + if 'likes' in open(outbox_post_filename, encoding='utf-8').read(): break time.sleep(1) @@ -1426,7 +1426,7 @@ def test_post_message_between_servers(base_dir: str) -> None: if alice_post_json: pprint(alice_post_json) - assert 'likes' in open(outbox_post_filename).read() + assert 'likes' in open(outbox_post_filename, encoding='utf-8').read() print('\n\n*******************************************************') print("Bob reacts to Alice's post") @@ -1440,8 +1440,8 @@ def test_post_message_between_servers(base_dir: str) -> None: True, __version__, signing_priv_key_pem, bob_domain, None, None) - for i in range(20): - if 'reactions' in open(outbox_post_filename).read(): + for _ in range(20): + if 'reactions' in open(outbox_post_filename, encoding='utf-8').read(): break time.sleep(1) @@ -1450,7 +1450,7 @@ def test_post_message_between_servers(base_dir: str) -> None: pprint(alice_post_json) # TODO: fix reactions unit test -# assert 'reactions' in open(outbox_post_filename).read() +# assert 'reactions' in open(outbox_post_filename, encoding='utf-8').read() print('\n\n*******************************************************') print("Bob repeats Alice's post") @@ -1478,7 +1478,7 @@ def test_post_message_between_servers(base_dir: str) -> None: bob_domain, None, None) announce_message_arrived = False outbox_message_arrived = False - for i in range(20): + for _ in range(20): time.sleep(1) if not os.path.isdir(inbox_path): continue @@ -1639,12 +1639,15 @@ def test_follow_between_servers(base_dir: str) -> None: alice_domain, alice_port) assert 'alice@' + alice_domain in open(bob_dir + '/accounts/bob@' + bob_domain + - '/followers.txt').read() + '/followers.txt', + encoding='utf-8').read() assert 'bob@' + bob_domain in open(alice_dir + '/accounts/alice@' + - alice_domain + '/following.txt').read() + alice_domain + '/following.txt', + encoding='utf-8').read() assert 'bob@' + bob_domain in open(alice_dir + '/accounts/alice@' + alice_domain + - '/followingCalendar.txt').read() + '/followingCalendar.txt', + encoding='utf-8').read() assert not is_group_actor(alice_dir, bob_actor, alice_person_cache) assert not is_group_account(alice_dir, 'alice', alice_domain) @@ -1679,7 +1682,7 @@ def test_follow_between_servers(base_dir: str) -> None: queue_path = bob_dir + '/accounts/bob@' + bob_domain + '/queue' inbox_path = bob_dir + '/accounts/bob@' + bob_domain + '/inbox' alice_message_arrived = False - for i in range(20): + for _ in range(20): time.sleep(1) if os.path.isdir(inbox_path): if len([name for name in os.listdir(inbox_path) @@ -1860,12 +1863,15 @@ def test_shared_items_federation(base_dir: str) -> None: alice_domain, alice_port) assert 'alice@' + alice_domain in open(bob_dir + '/accounts/bob@' + bob_domain + - '/followers.txt').read() + '/followers.txt', + encoding='utf-8').read() assert 'bob@' + bob_domain in open(alice_dir + '/accounts/alice@' + - alice_domain + '/following.txt').read() + alice_domain + '/following.txt', + encoding='utf-8').read() assert 'bob@' + bob_domain in open(alice_dir + '/accounts/alice@' + alice_domain + - '/followingCalendar.txt').read() + '/followingCalendar.txt', + encoding='utf-8').read() assert not is_group_actor(alice_dir, bob_actor, alice_person_cache) assert not is_group_account(bob_dir, 'bob', bob_domain) @@ -2316,20 +2322,22 @@ def test_group_follow(base_dir: str) -> None: assert valid_inbox(testgroup_dir, 'testgroup', testgroup_domain) assert valid_inbox_filenames(testgroup_dir, 'testgroup', testgroup_domain, alice_domain, alice_port) - assert 'alice@' + alice_domain in open(testgroup_followers_filename).read() + assert 'alice@' + alice_domain in open(testgroup_followers_filename, + encoding='utf-8').read() assert '!alice@' + alice_domain not in \ - open(testgroup_followers_filename).read() + open(testgroup_followers_filename, encoding='utf-8').read() testgroup_webfinger_filename = \ testgroup_dir + '/wfendpoints/testgroup@' + \ testgroup_domain + ':' + str(testgroupPort) + '.json' assert os.path.isfile(testgroup_webfinger_filename) - assert 'acct:testgroup@' in open(testgroup_webfinger_filename).read() + assert 'acct:testgroup@' in open(testgroup_webfinger_filename, + encoding='utf-8').read() print('acct: exists within the webfinger endpoint for testgroup') testgroup_handle = 'testgroup@' + testgroup_domain following_str = '' - with open(alice_following_filename, 'r') as fp_foll: + with open(alice_following_filename, 'r', encoding='utf-8') as fp_foll: following_str = fp_foll.read() print('Alice following.txt:\n\n' + following_str) if '!testgroup' not in following_str: @@ -2339,8 +2347,10 @@ def test_group_follow(base_dir: str) -> None: assert not is_group_account(alice_dir, 'alice', alice_domain) assert is_group_account(testgroup_dir, 'testgroup', testgroup_domain) assert '!testgroup' in following_str - assert testgroup_handle in open(alice_following_filename).read() - assert testgroup_handle in open(alice_following_calendar_filename).read() + assert testgroup_handle in open(alice_following_filename, + encoding='utf-8').read() + assert testgroup_handle in open(alice_following_calendar_filename, + encoding='utf-8').read() print('\n\n*********************************************************') print('Alice follows the test group') @@ -2394,20 +2404,22 @@ def test_group_follow(base_dir: str) -> None: assert valid_inbox(testgroup_dir, 'testgroup', testgroup_domain) assert valid_inbox_filenames(testgroup_dir, 'testgroup', testgroup_domain, bob_domain, bob_port) - assert 'bob@' + bob_domain in open(testgroup_followers_filename).read() + assert 'bob@' + bob_domain in open(testgroup_followers_filename, + encoding='utf-8').read() assert '!bob@' + bob_domain not in \ - open(testgroup_followers_filename).read() + open(testgroup_followers_filename, encoding='utf-8').read() testgroup_webfinger_filename = \ testgroup_dir + '/wfendpoints/testgroup@' + \ testgroup_domain + ':' + str(testgroupPort) + '.json' assert os.path.isfile(testgroup_webfinger_filename) - assert 'acct:testgroup@' in open(testgroup_webfinger_filename).read() + assert 'acct:testgroup@' in open(testgroup_webfinger_filename, + encoding='utf-8').read() print('acct: exists within the webfinger endpoint for testgroup') testgroup_handle = 'testgroup@' + testgroup_domain following_str = '' - with open(bob_following_filename, 'r') as fp_foll: + with open(bob_following_filename, 'r', encoding='utf-8') as fp_foll: following_str = fp_foll.read() print('Bob following.txt:\n\n' + following_str) if '!testgroup' not in following_str: @@ -2415,8 +2427,10 @@ def test_group_follow(base_dir: str) -> None: testgroup_domain + ':' + str(testgroupPort)) assert is_group_actor(bob_dir, testgroup_actor, bob_person_cache) assert '!testgroup' in following_str - assert testgroup_handle in open(bob_following_filename).read() - assert testgroup_handle in open(bob_following_calendar_filename).read() + assert testgroup_handle in open(bob_following_filename, + encoding='utf-8').read() + assert testgroup_handle in open(bob_following_calendar_filename, + encoding='utf-8').read() print('Bob follows the test group') print('\n\n*********************************************************') @@ -2734,7 +2748,8 @@ def _test_follows(base_dir: str) -> None: federation_list, False, False) account_dir = acct_dir(base_dir, nickname, domain) - with open(account_dir + '/following.txt', 'r') as fp_foll: + with open(account_dir + '/following.txt', 'r', + encoding='utf-8') as fp_foll: domain_found = False for following_domain in fp_foll: test_domain = following_domain.split('@')[1] @@ -2772,7 +2787,8 @@ def _test_follows(base_dir: str) -> None: federation_list, False, False) account_dir = acct_dir(base_dir, nickname, domain) - with open(account_dir + '/followers.txt', 'r') as fp_foll: + with open(account_dir + '/followers.txt', 'r', + encoding='utf-8') as fp_foll: for follower_domain in fp_foll: test_domain = follower_domain.split('@')[1] test_domain = test_domain.replace('\n', '').replace('\r', '') @@ -3030,8 +3046,8 @@ def test_client_to_server(base_dir: str): print('Bob follows the calendar of Alice') following_cal_path = \ bob_dir + '/accounts/bob@' + bob_domain + '/followingCalendar.txt' - with open(following_cal_path, 'w+') as fp: - fp.write('alice@' + alice_domain + '\n') + with open(following_cal_path, 'w+', encoding='utf-8') as fp_foll: + fp_foll.write('alice@' + alice_domain + '\n') print('\n\n*******************************************************') print('EVENT: Alice sends to Bob via c2s') @@ -3172,11 +3188,13 @@ def test_client_to_server(base_dir: str): for _ in range(10): if os.path.isfile(bob_followers_filename): if 'alice@' + alice_domain + ':' + str(alice_port) in \ - open(bob_followers_filename).read(): + open(bob_followers_filename, + encoding='utf-8').read(): if os.path.isfile(alice_following_filename) and \ os.path.isfile(alice_petnames_filename): if 'bob@' + bob_domain + ':' + str(bob_port) in \ - open(alice_following_filename).read(): + open(alice_following_filename, + encoding='utf-8').read(): break time.sleep(1) @@ -3184,15 +3202,15 @@ def test_client_to_server(base_dir: str): assert os.path.isfile(alice_following_filename) assert os.path.isfile(alice_petnames_filename) assert 'bob bob@' + bob_domain in \ - open(alice_petnames_filename).read() + open(alice_petnames_filename, encoding='utf-8').read() print('alice@' + alice_domain + ':' + str(alice_port) + ' in ' + bob_followers_filename) assert 'alice@' + alice_domain + ':' + str(alice_port) in \ - open(bob_followers_filename).read() + open(bob_followers_filename, encoding='utf-8').read() print('bob@' + bob_domain + ':' + str(bob_port) + ' in ' + alice_following_filename) assert 'bob@' + bob_domain + ':' + str(bob_port) in \ - open(alice_following_filename).read() + open(alice_following_filename, encoding='utf-8').read() assert valid_inbox(bob_dir, 'bob', bob_domain) assert valid_inbox_filenames(bob_dir, 'bob', bob_domain, alice_domain, alice_port) @@ -3210,20 +3228,21 @@ def test_client_to_server(base_dir: str): '/followers.txt'): if 'bob@' + bob_domain + ':' + str(bob_port) in \ open(alice_dir + '/accounts/alice@' + alice_domain + - '/followers.txt').read(): + '/followers.txt', encoding='utf-8').read(): if os.path.isfile(bob_dir + '/accounts/bob@' + bob_domain + '/following.txt'): alice_handle_str = \ 'alice@' + alice_domain + ':' + str(alice_port) if alice_handle_str in \ open(bob_dir + '/accounts/bob@' + bob_domain + - '/following.txt').read(): + '/following.txt', encoding='utf-8').read(): if os.path.isfile(bob_dir + '/accounts/bob@' + bob_domain + '/followingCalendar.txt'): if alice_handle_str in \ open(bob_dir + '/accounts/bob@' + bob_domain + - '/followingCalendar.txt').read(): + '/followingCalendar.txt', + encoding='utf-8').read(): break time.sleep(1) @@ -3233,9 +3252,10 @@ def test_client_to_server(base_dir: str): '/following.txt') assert 'bob@' + bob_domain + ':' + str(bob_port) in \ open(alice_dir + '/accounts/alice@' + alice_domain + - '/followers.txt').read() + '/followers.txt', encoding='utf-8').read() assert 'alice@' + alice_domain + ':' + str(alice_port) in \ - open(bob_dir + '/accounts/bob@' + bob_domain + '/following.txt').read() + open(bob_dir + '/accounts/bob@' + bob_domain + '/following.txt', + encoding='utf-8').read() session_bob = create_session(proxy_type) password = 'bobpass' @@ -3439,18 +3459,18 @@ def test_client_to_server(base_dir: str): True, __version__, signing_priv_key_pem) for _ in range(10): if 'alice@' + alice_domain + ':' + str(alice_port) not in \ - open(bob_followers_filename).read(): + open(bob_followers_filename, encoding='utf-8').read(): if 'bob@' + bob_domain + ':' + str(bob_port) not in \ - open(alice_following_filename).read(): + open(alice_following_filename, encoding='utf-8').read(): break time.sleep(1) assert os.path.isfile(bob_followers_filename) assert os.path.isfile(alice_following_filename) assert 'alice@' + alice_domain + ':' + str(alice_port) \ - not in open(bob_followers_filename).read() + not in open(bob_followers_filename, encoding='utf-8').read() assert 'bob@' + bob_domain + ':' + str(bob_port) \ - not in open(alice_following_filename).read() + not in open(alice_following_filename, encoding='utf-8').read() assert valid_inbox(bob_dir, 'bob', bob_domain) assert valid_inbox_filenames(bob_dir, 'bob', bob_domain, alice_domain, alice_port) @@ -4228,7 +4248,7 @@ def _test_translation_labels() -> None: if source_file.startswith('flycheck_'): continue source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: continue @@ -4477,7 +4497,7 @@ def _test_hashtag_rules(): tree = hashtag_rule_tree(operators, conditions_str, tags_in_conditions, moderated) assert tree is None - assert tags_in_conditions == [] + assert not tags_in_conditions hashtags = ['#foo'] assert not hashtag_rule_resolve(tree, hashtags, moderated, content, url) @@ -4833,7 +4853,7 @@ def _diagram_groups(include_groups: [], for group_name in include_groups: filename += '_' + group_name.replace(' ', '-') filename += '.dot' - with open(filename, 'w+') as fp_graph: + with open(filename, 'w+', encoding='utf-8') as fp_graph: fp_graph.write(call_graph_str) print('Graph saved to ' + filename) print('Plot using: ' + @@ -4853,7 +4873,7 @@ def _test_post_variable_names(): if source_file.startswith('flycheck_'): continue source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: continue @@ -4888,7 +4908,7 @@ def _test_config_param_names(): if source_file.startswith('flycheck_'): continue source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: continue @@ -4936,7 +4956,7 @@ def _test_source_contains_no_tabs(): if source_file.startswith('flycheck_'): continue source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: continue @@ -4959,7 +4979,7 @@ def _test_checkbox_names(): if source_file.startswith('flycheck_'): continue source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: continue @@ -4999,7 +5019,7 @@ def _test_post_field_names(source_file: str, fieldnames: []): fnames.append(field + '.get') source_str = '' - with open(source_file, 'r') as file_source: + with open(source_file, 'r', encoding='utf-8') as file_source: source_str = file_source.read() if not source_str: return @@ -5082,12 +5102,12 @@ def _test_thread_functions(): 'functions': [] } source_str = '' - with open(source_file, 'r') as fp_src: + with open(source_file, 'r', encoding='utf-8') as fp_src: source_str = fp_src.read() modules[mod_name]['source'] = source_str if 'thread_with_trace(' in source_str: threads_called_in_modules.append(mod_name) - with open(source_file, 'r') as fp_src: + with open(source_file, 'r', encoding='utf-8') as fp_src: lines = fp_src.readlines() modules[mod_name]['lines'] = lines @@ -5118,7 +5138,7 @@ def _test_thread_functions(): if def_str not in modules[mod_name]['source']: print(mod_name + ' ' + first_parameter) print(import_str + ' not in ' + mod_name) - assert(False) + assert False if def_str in modules[mod_name]['source']: defininition_module = mod_name else: @@ -5172,7 +5192,7 @@ def _test_thread_functions(): ' were given in module ' + mod_name) print(str(definition_function_args_list)) print(str(calling_function_args_list)) - assert(False) + assert False ctr += 1 @@ -5197,10 +5217,10 @@ def _test_functions(): 'functions': [] } source_str = '' - with open(source_file, 'r') as fp_src: + with open(source_file, 'r', encoding='utf-8') as fp_src: source_str = fp_src.read() modules[mod_name]['source'] = source_str - with open(source_file, 'r') as fp_src: + with open(source_file, 'r', encoding='utf-8') as fp_src: lines = fp_src.readlines() modules[mod_name]['lines'] = lines line_count = 0 @@ -5313,7 +5333,8 @@ def _test_functions(): 'sed -i "s|' + name + '|' + snake_case_name + '|g" *.py\n' ctr += 1 print(function_names_str + '\n') - with open('scripts/bad_function_names.sh', 'w+') as fn_file: + with open('scripts/bad_function_names.sh', 'w+', + encoding='utf-8') as fn_file: fn_file.write(function_names_sh) assert False @@ -6246,7 +6267,7 @@ def _test_spoofed_geolocation() -> None: kml_str += '\n' kml_str += '' - with open('unittest_decoy.kml', 'w+') as kmlfile: + with open('unittest_decoy.kml', 'w+', encoding='utf-8') as kmlfile: kmlfile.write(kml_str) diff --git a/theme.py b/theme.py index 3ddd63353..aefdec70c 100644 --- a/theme.py +++ b/theme.py @@ -41,7 +41,8 @@ def import_theme(base_dir: str, filename: str) -> bool: ' missing from imported theme') return False new_theme_name = None - with open(temp_theme_dir + '/name.txt', 'r') as fp_theme: + with open(temp_theme_dir + '/name.txt', 'r', + encoding='utf-8') as fp_theme: new_theme_name = fp_theme.read().replace('\n', '').replace('\r', '') if len(new_theme_name) > 20: print('WARN: Imported theme name is too long') @@ -65,7 +66,7 @@ def import_theme(base_dir: str, filename: str) -> bool: default_themes_filename = base_dir + '/defaultthemes.txt' if os.path.isfile(default_themes_filename): if new_theme_name.title() + '\n' in \ - open(default_themes_filename).read(): + open(default_themes_filename, encoding='utf-8').read(): new_theme_name = new_theme_name + '2' theme_dir = base_dir + '/theme/' + new_theme_name @@ -349,7 +350,7 @@ def _set_theme_from_dict(base_dir: str, name: str, if not os.path.isfile(template_filename): continue - with open(template_filename, 'r') as cssfile: + with open(template_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() for param_name, param_value in theme_params.items(): if param_name == 'newswire-publish-icon': @@ -384,7 +385,7 @@ def _set_theme_from_dict(base_dir: str, name: str, continue css = set_css_param(css, param_name, param_value) filename = base_dir + '/' + filename - with open(filename, 'w+') as cssfile: + with open(filename, 'w+', encoding='utf-8') as cssfile: cssfile.write(css) screen_name = ( @@ -404,10 +405,10 @@ def _set_background_format(base_dir: str, name: str, css_filename = base_dir + '/' + background_type + '.css' if not os.path.isfile(css_filename): return - with open(css_filename, 'r') as cssfile: + with open(css_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() css = css.replace('background.jpg', 'background.' + extension) - with open(css_filename, 'w+') as cssfile2: + with open(css_filename, 'w+', encoding='utf-8') as cssfile2: cssfile2.write(css) @@ -419,18 +420,18 @@ def enable_grayscale(base_dir: str) -> None: template_filename = base_dir + '/' + filename if not os.path.isfile(template_filename): continue - with open(template_filename, 'r') as cssfile: + with open(template_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() if 'grayscale' not in css: css = \ css.replace('body, html {', 'body, html {\n filter: grayscale(100%);') filename = base_dir + '/' + filename - with open(filename, 'w+') as cssfile: + with open(filename, 'w+', encoding='utf-8') as cssfile: cssfile.write(css) grayscale_filename = base_dir + '/accounts/.grayscale' if not os.path.isfile(grayscale_filename): - with open(grayscale_filename, 'w+') as grayfile: + with open(grayscale_filename, 'w+', encoding='utf-8') as grayfile: grayfile.write(' ') @@ -442,13 +443,13 @@ def disable_grayscale(base_dir: str) -> None: template_filename = base_dir + '/' + filename if not os.path.isfile(template_filename): continue - with open(template_filename, 'r') as cssfile: + with open(template_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() if 'grayscale' in css: css = \ css.replace('\n filter: grayscale(100%);', '') filename = base_dir + '/' + filename - with open(filename, 'w+') as cssfile: + with open(filename, 'w+', encoding='utf-8') as cssfile: cssfile.write(css) grayscale_filename = base_dir + '/accounts/.grayscale' if os.path.isfile(grayscale_filename): @@ -467,7 +468,7 @@ def _set_dyslexic_font(base_dir: str) -> bool: template_filename = base_dir + '/' + filename if not os.path.isfile(template_filename): continue - with open(template_filename, 'r') as cssfile: + with open(template_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() css = \ set_css_param(css, "*src", @@ -475,7 +476,7 @@ def _set_dyslexic_font(base_dir: str) -> bool: "') format('woff2')") css = set_css_param(css, "*font-family", "'OpenDyslexic'") filename = base_dir + '/' + filename - with open(filename, 'w+') as cssfile: + with open(filename, 'w+', encoding='utf-8') as cssfile: cssfile.write(css) return False @@ -504,7 +505,7 @@ def _set_custom_font(base_dir: str): template_filename = base_dir + '/' + filename if not os.path.isfile(template_filename): continue - with open(template_filename, 'r') as cssfile: + with open(template_filename, 'r', encoding='utf-8') as cssfile: css = cssfile.read() css = \ set_css_param(css, "*src", @@ -515,7 +516,7 @@ def _set_custom_font(base_dir: str): css = set_css_param(css, "*font-family", "'CustomFont'") css = set_css_param(css, "header-font", "'CustomFont'") filename = base_dir + '/' + filename - with open(filename, 'w+') as cssfile: + with open(filename, 'w+', encoding='utf-8') as cssfile: cssfile.write(css) @@ -630,7 +631,8 @@ def get_text_mode_banner(base_dir: str) -> str: """ text_mode_banner_filename = base_dir + '/accounts/banner.txt' if os.path.isfile(text_mode_banner_filename): - with open(text_mode_banner_filename, 'r') as fp_text: + with open(text_mode_banner_filename, 'r', + encoding='utf-8') as fp_text: banner_str = fp_text.read() if banner_str: return banner_str.replace('\n', '
') @@ -644,7 +646,7 @@ def get_text_mode_logo(base_dir: str) -> str: if not os.path.isfile(text_mode_logo_filename): text_mode_logo_filename = base_dir + '/img/logo.txt' - with open(text_mode_logo_filename, 'r') as fp_text: + with open(text_mode_logo_filename, 'r', encoding='utf-8') as fp_text: logo_str = fp_text.read() if logo_str: return logo_str.replace('\n', '
') @@ -846,7 +848,7 @@ def _set_clear_cache_flag(base_dir: str) -> None: if not os.path.isdir(base_dir + '/accounts'): return flag_filename = base_dir + '/accounts/.clear_cache' - with open(flag_filename, 'w+') as fp_flag: + with open(flag_filename, 'w+', encoding='utf-8') as fp_flag: fp_flag.write('\n') @@ -924,7 +926,7 @@ def update_default_themes_list(base_dir: str) -> None: """ theme_names = get_themes_list(base_dir) default_themes_filename = base_dir + '/defaultthemes.txt' - with open(default_themes_filename, 'w+') as fp_def: + with open(default_themes_filename, 'w+', encoding='utf-8') as fp_def: for name in theme_names: fp_def.write(name + '\n') @@ -938,7 +940,7 @@ def scan_themes_for_scripts(base_dir: str) -> bool: continue svg_filename = os.path.join(subdir, fname) content = '' - with open(svg_filename, 'r') as fp_svg: + with open(svg_filename, 'r', encoding='utf-8') as fp_svg: content = fp_svg.read() svg_dangerous = dangerous_svg(content, False) if svg_dangerous: diff --git a/threads.py b/threads.py index 02ae94684..7d8974d96 100644 --- a/threads.py +++ b/threads.py @@ -156,7 +156,7 @@ def remove_dormant_threads(base_dir: str, threads_list: [], debug: bool, if debug: send_log_filename = base_dir + '/send.csv' try: - with open(send_log_filename, 'a+') as fp_log: + with open(send_log_filename, 'a+', encoding='utf-8') as fp_log: fp_log.write(curr_time.strftime("%Y-%m-%dT%H:%M:%SZ") + ',' + str(no_of_active_threads) + ',' + str(len(threads_list)) + '\n') diff --git a/utils.py b/utils.py index 98f0136e2..5d49d7379 100644 --- a/utils.py +++ b/utils.py @@ -197,7 +197,8 @@ def refresh_newswire(base_dir: str): refresh_newswire_filename = base_dir + '/accounts/.refresh_newswire' if os.path.isfile(refresh_newswire_filename): return - with open(refresh_newswire_filename, 'w+') as refresh_file: + with open(refresh_newswire_filename, 'w+', + encoding='utf-8') as refresh_file: refresh_file.write('\n') @@ -328,7 +329,8 @@ def is_dormant(base_dir: str, nickname: str, domain: str, actor: str, days_since_epoch_str = None try: - with open(last_seen_filename, 'r') as last_seen_file: + with open(last_seen_filename, 'r', + encoding='utf-8') as last_seen_file: days_since_epoch_str = last_seen_file.read() except OSError: print('EX: failed to read last seen ' + last_seen_filename) @@ -358,7 +360,7 @@ def is_editor(base_dir: str, nickname: str) -> bool: return True return False - with open(editors_file, 'r') as editors: + with open(editors_file, 'r', encoding='utf-8') as editors: lines = editors.readlines() if len(lines) == 0: admin_name = get_config_param(base_dir, 'admin') @@ -384,7 +386,7 @@ def is_artist(base_dir: str, nickname: str) -> bool: return True return False - with open(artists_file, 'r') as artists: + with open(artists_file, 'r', encoding='utf-8') as artists: lines = artists.readlines() if len(lines) == 0: admin_name = get_config_param(base_dir, 'admin') @@ -601,7 +603,7 @@ def is_suspended(base_dir: str, nickname: str) -> bool: suspended_filename = base_dir + '/accounts/suspended.txt' if os.path.isfile(suspended_filename): - with open(suspended_filename, 'r') as susp_file: + with open(suspended_filename, 'r', encoding='utf-8') as susp_file: lines = susp_file.readlines() for suspended in lines: if suspended.strip('\n').strip('\r') == nickname: @@ -619,7 +621,7 @@ def get_followers_list(base_dir: str, if not os.path.isfile(filename): return [] - with open(filename, 'r') as foll_file: + with open(filename, 'r', encoding='utf-8') as foll_file: lines = foll_file.readlines() for i, _ in enumerate(lines): lines[i] = lines[i].strip() @@ -648,7 +650,7 @@ def get_followers_of_person(base_dir: str, continue if not os.path.isfile(filename): continue - with open(filename, 'r') as followingfile: + with open(filename, 'r', encoding='utf-8') as followingfile: for following_handle in followingfile: following_handle2 = following_handle.replace('\n', '') following_handle2 = following_handle2.replace('\r', '') @@ -724,7 +726,7 @@ def save_json(json_object: {}, filename: str) -> bool: tries = 0 while tries < 5: try: - with open(filename, 'w+') as json_file: + with open(filename, 'w+', encoding='utf-8') as json_file: json_file.write(json.dumps(json_object)) return True except OSError: @@ -743,7 +745,7 @@ def load_json(filename: str, delay_sec: int = 2, max_tries: int = 5) -> {}: tries = 0 while tries < max_tries: try: - with open(filename, 'r') as json_file: + with open(filename, 'r', encoding='utf-8') as json_file: data = json_file.read() json_object = json.loads(data) break @@ -766,7 +768,7 @@ def load_json_onionify(filename: str, domain: str, onion_domain: str, tries = 0 while tries < 5: try: - with open(filename, 'r') as json_file: + with open(filename, 'r', encoding='utf-8') as json_file: data = json_file.read() if data: data = data.replace(domain, onion_domain) @@ -1262,11 +1264,11 @@ def _set_default_pet_name(base_dir: str, nickname: str, domain: str, follow_nickname + '@' + follow_domain + '\n' if not os.path.isfile(petnames_filename): # if there is no existing petnames lookup file - with open(petnames_filename, 'w+') as petnames_file: + with open(petnames_filename, 'w+', encoding='utf-8') as petnames_file: petnames_file.write(petname_lookup_entry) return - with open(petnames_filename, 'r') as petnames_file: + with open(petnames_filename, 'r', encoding='utf-8') as petnames_file: petnames_str = petnames_file.read() if petnames_str: petnames_list = petnames_str.split('\n') @@ -1275,7 +1277,7 @@ def _set_default_pet_name(base_dir: str, nickname: str, domain: str, # petname already exists return # petname doesn't already exist - with open(petnames_filename, 'a+') as petnames_file: + with open(petnames_filename, 'a+', encoding='utf-8') as petnames_file: petnames_file.write(petname_lookup_entry) @@ -1318,15 +1320,18 @@ def follow_person(base_dir: str, nickname: str, domain: str, # was this person previously unfollowed? unfollowed_filename = base_dir + '/accounts/' + handle + '/unfollowed.txt' if os.path.isfile(unfollowed_filename): - if handle_to_follow in open(unfollowed_filename).read(): + if handle_to_follow in open(unfollowed_filename, + encoding='utf-8').read(): # remove them from the unfollowed file new_lines = '' - with open(unfollowed_filename, 'r') as unfoll_file: + with open(unfollowed_filename, 'r', + encoding='utf-8') as unfoll_file: lines = unfoll_file.readlines() for line in lines: if handle_to_follow not in line: new_lines += line - with open(unfollowed_filename, 'w+') as unfoll_file: + with open(unfollowed_filename, 'w+', + encoding='utf-8') as unfoll_file: unfoll_file.write(new_lines) if not os.path.isdir(base_dir + '/accounts'): @@ -1336,13 +1341,13 @@ def follow_person(base_dir: str, nickname: str, domain: str, handle_to_follow = '!' + handle_to_follow filename = base_dir + '/accounts/' + handle + '/' + follow_file if os.path.isfile(filename): - if handle_to_follow in open(filename).read(): + if handle_to_follow in open(filename, encoding='utf-8').read(): if debug: print('DEBUG: follow already exists') return True # prepend to follow file try: - with open(filename, 'r+') as foll_file: + with open(filename, 'r+', encoding='utf-8') as foll_file: content = foll_file.read() if handle_to_follow + '\n' not in content: foll_file.seek(0, 0) @@ -1358,7 +1363,7 @@ def follow_person(base_dir: str, nickname: str, domain: str, ' creating new following file to follow ' + handle_to_follow + ', filename is ' + filename) - with open(filename, 'w+') as foll_file: + with open(filename, 'w+', encoding='utf-8') as foll_file: foll_file.write(handle_to_follow + '\n') if follow_file.endswith('following.txt'): @@ -1430,7 +1435,7 @@ def locate_news_arrival(base_dir: str, domain: str, account_dir = base_dir + '/accounts/news@' + domain + '/' post_filename = account_dir + 'outbox/' + post_url if os.path.isfile(post_filename): - with open(post_filename, 'r') as arrival_file: + with open(post_filename, 'r', encoding='utf-8') as arrival_file: arrival = arrival_file.read() if arrival: arrival_date = \ @@ -1537,7 +1542,8 @@ def get_reply_interval_hours(base_dir: str, nickname: str, domain: str, reply_interval_filename = \ acct_dir(base_dir, nickname, domain) + '/.reply_interval_hours' if os.path.isfile(reply_interval_filename): - with open(reply_interval_filename, 'r') as interval_file: + with open(reply_interval_filename, 'r', + encoding='utf-8') as interval_file: hours_str = interval_file.read() if hours_str.isdigit(): return int(hours_str) @@ -1553,7 +1559,8 @@ def set_reply_interval_hours(base_dir: str, nickname: str, domain: str, reply_interval_filename = \ acct_dir(base_dir, nickname, domain) + '/.reply_interval_hours' try: - with open(reply_interval_filename, 'w+') as interval_file: + with open(reply_interval_filename, 'w+', + encoding='utf-8') as interval_file: interval_file.write(str(reply_interval_hours)) return True except OSError: @@ -1641,10 +1648,12 @@ def remove_moderation_post_from_index(base_dir: str, post_url: str, if not os.path.isfile(moderation_index_file): return post_id = remove_id_ending(post_url) - if post_id in open(moderation_index_file).read(): - with open(moderation_index_file, 'r') as file1: + if post_id in open(moderation_index_file, encoding='utf-8').read(): + with open(moderation_index_file, 'r', + encoding='utf-8') as file1: lines = file1.readlines() - with open(moderation_index_file, 'w+') as file2: + with open(moderation_index_file, 'w+', + encoding='utf-8') as file2: for line in lines: if line.strip("\n").strip("\r") != post_id: file2.write(line) @@ -1670,7 +1679,7 @@ def _is_reply_to_blog_post(base_dir: str, nickname: str, domain: str, return False post_id = remove_id_ending(post_json_object['object']['inReplyTo']) post_id = post_id.replace('/', '#') - if post_id in open(blogs_index_filename).read(): + if post_id in open(blogs_index_filename, encoding='utf-8').read(): return True return False @@ -1686,7 +1695,7 @@ def _delete_post_remove_replies(base_dir: str, nickname: str, domain: str, return if debug: print('DEBUG: removing replies to ' + post_filename) - with open(replies_filename, 'r') as replies_file: + with open(replies_filename, 'r', encoding='utf-8') as replies_file: for reply_id in replies_file: reply_file = locate_post(base_dir, nickname, domain, reply_id) if not reply_file: @@ -1711,7 +1720,8 @@ def _is_bookmarked(base_dir: str, nickname: str, domain: str, acct_dir(base_dir, nickname, domain) + '/bookmarks.index' if os.path.isfile(bookmarks_index_filename): bookmark_index = post_filename.split('/')[-1] + '\n' - if bookmark_index in open(bookmarks_index_filename).read(): + if bookmark_index in open(bookmarks_index_filename, + encoding='utf-8').read(): return True return False @@ -1815,7 +1825,7 @@ def _delete_hashtags_on_post(base_dir: str, post_json_object: {}) -> None: continue # remove post_id from the tag index file lines = None - with open(tag_index_filename, 'r') as index_file: + with open(tag_index_filename, 'r', encoding='utf-8') as index_file: lines = index_file.readlines() if not lines: continue @@ -1834,7 +1844,8 @@ def _delete_hashtags_on_post(base_dir: str, post_json_object: {}) -> None: 'unable to delete tag index ' + str(tag_index_filename)) else: # write the new hashtag index without the given post in it - with open(tag_index_filename, 'w+') as index_file: + with open(tag_index_filename, 'w+', + encoding='utf-8') as index_file: index_file.write(newlines) @@ -1857,13 +1868,13 @@ def _delete_conversation_post(base_dir: str, nickname: str, domain: str, if not os.path.isfile(conversation_filename): return False conversation_str = '' - with open(conversation_filename, 'r') as conv_file: + with open(conversation_filename, 'r', encoding='utf-8') as conv_file: conversation_str = conv_file.read() if post_id + '\n' not in conversation_str: return False conversation_str = conversation_str.replace(post_id + '\n', '') if conversation_str: - with open(conversation_filename, 'w+') as conv_file: + with open(conversation_filename, 'w+', encoding='utf-8') as conv_file: conv_file.write(conversation_str) else: if os.path.isfile(conversation_filename + '.muted'): @@ -2175,7 +2186,8 @@ def no_of_active_accounts_monthly(base_dir: str, months: int) -> bool: base_dir + '/accounts/' + account + '/.lastUsed' if not os.path.isfile(last_used_filename): continue - with open(last_used_filename, 'r') as last_used_file: + with open(last_used_filename, 'r', + encoding='utf-8') as last_used_file: last_used = last_used_file.read() if last_used.isdigit(): time_diff = (curr_time - int(last_used)) @@ -2334,7 +2346,7 @@ def get_css(base_dir: str, css_filename: str, css_cache: {}) -> str: # file hasn't changed, so return the version in the cache return css_cache[css_filename][1] - with open(css_filename, 'r') as fp_css: + with open(css_filename, 'r', encoding='utf-8') as fp_css: css = fp_css.read() if css_cache.get(css_filename): # alter the cache contents @@ -2394,7 +2406,7 @@ def _search_virtual_box_posts(base_dir: str, nickname: str, domain: str, search_words = [search_str] res = [] - with open(index_filename, 'r') as index_file: + with open(index_filename, 'r', encoding='utf-8') as index_file: post_filename = 'start' while post_filename: post_filename = index_file.readline() @@ -2405,7 +2417,7 @@ def _search_virtual_box_posts(base_dir: str, nickname: str, domain: str, post_filename = path + '/' + post_filename.strip() if not os.path.isfile(post_filename): continue - with open(post_filename, 'r') as post_file: + with open(post_filename, 'r', encoding='utf-8') as post_file: data = post_file.read().lower() not_found = False @@ -2449,7 +2461,7 @@ def search_box_posts(base_dir: str, nickname: str, domain: str, for root, _, fnames in os.walk(path): for fname in fnames: file_path = os.path.join(root, fname) - with open(file_path, 'r') as post_file: + with open(file_path, 'r', encoding='utf-8') as post_file: data = post_file.read().lower() not_found = False @@ -2892,7 +2904,8 @@ def reject_post_id(base_dir: str, nickname: str, domain: str, if recent_posts_cache['html'].get(post_url): del recent_posts_cache['html'][post_url] - with open(post_filename + '.reject', 'w+') as reject_file: + with open(post_filename + '.reject', 'w+', + encoding='utf-8') as reject_file: reject_file.write('\n') @@ -3011,7 +3024,8 @@ def dm_allowed_from_domain(base_dir: str, acct_dir(base_dir, nickname, domain) + '/dmAllowedInstances.txt' if not os.path.isfile(dm_allowed_instances_file): return False - if sending_actor_domain + '\n' in open(dm_allowed_instances_file).read(): + if sending_actor_domain + '\n' in open(dm_allowed_instances_file, + encoding='utf-8').read(): return True return False @@ -3325,7 +3339,8 @@ def is_group_actor(base_dir: str, actor: str, person_cache: {}, if debug: print('Cached actor file not found ' + cached_actor_filename) return False - if '"type": "Group"' in open(cached_actor_filename).read(): + if '"type": "Group"' in open(cached_actor_filename, + encoding='utf-8').read(): if debug: print('Group type found in ' + cached_actor_filename) return True @@ -3338,7 +3353,8 @@ def is_group_account(base_dir: str, nickname: str, domain: str) -> bool: account_filename = acct_dir(base_dir, nickname, domain) + '.json' if not os.path.isfile(account_filename): return False - if '"type": "Group"' in open(account_filename).read(): + if '"type": "Group"' in open(account_filename, + encoding='utf-8').read(): return True return False @@ -3596,7 +3612,7 @@ def load_account_timezones(base_dir: str) -> {}: if not os.path.isfile(tz_filename): continue timezone = None - with open(tz_filename, 'r') as fp_timezone: + with open(tz_filename, 'r', encoding='utf-8') as fp_timezone: timezone = fp_timezone.read().strip() if timezone: nickname = acct.split('@')[0] @@ -3632,7 +3648,7 @@ def get_account_timezone(base_dir: str, nickname: str, domain: str) -> str: if not os.path.isfile(tz_filename): return None timezone = None - with open(tz_filename, 'r') as fp_timezone: + with open(tz_filename, 'r', encoding='utf-8') as fp_timezone: timezone = fp_timezone.read().strip() return timezone @@ -3644,7 +3660,7 @@ def set_account_timezone(base_dir: str, nickname: str, domain: str, tz_filename = \ base_dir + '/accounts/' + nickname + '@' + domain + '/timezone.txt' timezone = timezone.strip() - with open(tz_filename, 'w+') as fp_timezone: + with open(tz_filename, 'w+', encoding='utf-8') as fp_timezone: fp_timezone.write(timezone) diff --git a/webapp_about.py b/webapp_about.py index 6ef697750..a193606fb 100644 --- a/webapp_about.py +++ b/webapp_about.py @@ -32,7 +32,8 @@ def html_about(css_cache: {}, base_dir: str, http_prefix: str, about_text = 'Information about this instance goes here.' if os.path.isfile(base_dir + '/accounts/about.md'): - with open(base_dir + '/accounts/about.md', 'r') as fp_about: + with open(base_dir + '/accounts/about.md', 'r', + encoding='utf-8') as fp_about: about_text = markdown_to_html(fp_about.read()) about_form = '' diff --git a/webapp_column_left.py b/webapp_column_left.py index 97df269e7..94254b6e0 100644 --- a/webapp_column_left.py +++ b/webapp_column_left.py @@ -218,7 +218,7 @@ def get_left_column_content(base_dir: str, nickname: str, domain_full: str, links_file_contains_entries = False links_list = None if os.path.isfile(links_filename): - with open(links_filename, 'r') as fp_links: + with open(links_filename, 'r', encoding='utf-8') as fp_links: links_list = fp_links.readlines() if not front_page: diff --git a/webapp_column_right.py b/webapp_column_right.py index e7fae6d02..5e41fe570 100644 --- a/webapp_column_right.py +++ b/webapp_column_right.py @@ -372,7 +372,7 @@ def html_citations(base_dir: str, nickname: str, domain: str, citations_selected = [] if os.path.isfile(citations_filename): citations_separator = '#####' - with open(citations_filename, 'r') as fp_cit: + with open(citations_filename, 'r', encoding='utf-8') as fp_cit: citations = fp_cit.readlines() for line in citations: if citations_separator not in line: @@ -611,7 +611,7 @@ def html_edit_newswire(css_cache: {}, translate: {}, base_dir: str, path: str, newswire_filename = base_dir + '/accounts/newswire.txt' newswire_str = '' if os.path.isfile(newswire_filename): - with open(newswire_filename, 'r') as fp_news: + with open(newswire_filename, 'r', encoding='utf-8') as fp_news: newswire_str = fp_news.read() edit_newswire_form += \ @@ -633,7 +633,7 @@ def html_edit_newswire(css_cache: {}, translate: {}, base_dir: str, path: str, filter_filename = \ base_dir + '/accounts/news@' + domain + '/filters.txt' if os.path.isfile(filter_filename): - with open(filter_filename, 'r') as filterfile: + with open(filter_filename, 'r', encoding='utf-8') as filterfile: filter_str = filterfile.read() edit_newswire_form += \ @@ -649,7 +649,7 @@ def html_edit_newswire(css_cache: {}, translate: {}, base_dir: str, path: str, hashtag_rules_filename = \ base_dir + '/accounts/hashtagrules.txt' if os.path.isfile(hashtag_rules_filename): - with open(hashtag_rules_filename, 'r') as rulesfile: + with open(hashtag_rules_filename, 'r', encoding='utf-8') as rulesfile: hashtag_rules_str = rulesfile.read() edit_newswire_form += \ diff --git a/webapp_create_post.py b/webapp_create_post.py index 48b0e08a7..51dfcc54e 100644 --- a/webapp_create_post.py +++ b/webapp_create_post.py @@ -42,7 +42,8 @@ def _html_following_data_list(base_dir: str, nickname: str, acct_dir(base_dir, nickname, domain) + '/following.txt' msg = None if os.path.isfile(following_filename): - with open(following_filename, 'r') as following_file: + with open(following_filename, 'r', + encoding='utf-8') as following_file: msg = following_file.read() # add your own handle, so that you can send DMs # to yourself as reminders @@ -53,7 +54,8 @@ def _html_following_data_list(base_dir: str, nickname: str, acct_dir(base_dir, nickname, domain) + '/petnames.txt' if os.path.isfile(petnames_filename): following_list = [] - with open(petnames_filename, 'r') as petnames_file: + with open(petnames_filename, 'r', + encoding='utf-8') as petnames_file: pet_str = petnames_file.read() # extract each petname and append it petnames_list = pet_str.split('\n') @@ -322,7 +324,8 @@ def html_new_post(css_cache: {}, media_instance: bool, translate: {}, # custom report header with any additional instructions if os.path.isfile(base_dir + '/accounts/report.txt'): - with open(base_dir + '/accounts/report.txt', 'r') as file: + with open(base_dir + '/accounts/report.txt', 'r', + encoding='utf-8') as file: custom_report_text = file.read() if '

' not in custom_report_text: custom_report_text = \ @@ -359,7 +362,8 @@ def html_new_post(css_cache: {}, media_instance: bool, translate: {}, '\n' if os.path.isfile(base_dir + '/accounts/newpost.txt'): - with open(base_dir + '/accounts/newpost.txt', 'r') as file: + with open(base_dir + '/accounts/newpost.txt', 'r', + encoding='utf-8') as file: new_post_text = \ '

' + file.read() + '

\n' @@ -617,7 +621,7 @@ def html_new_post(css_cache: {}, media_instance: bool, translate: {}, translate['Citations'] + ':

\n' citations_str += '