diff --git a/auth.py b/auth.py index 94af04139..9d67ad972 100644 --- a/auth.py +++ b/auth.py @@ -16,6 +16,7 @@ import datetime from utils import is_system_account from utils import has_users_path from utils import text_in_file +from utils import remove_eol def _hash_password(password: str) -> str: @@ -78,9 +79,9 @@ def create_basic_auth_header(nickname: str, password: str) -> str: """This is only used by tests """ auth_str = \ - nickname.replace('\n', '').replace('\r', '') + \ + remove_eol(nickname) + \ ':' + \ - password.replace('\n', '').replace('\r', '') + remove_eol(password) return 'Basic ' + \ base64.b64encode(auth_str.encode('utf-8')).decode('utf-8') @@ -118,8 +119,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str, print('basic auth - attempted login using system account ' + nickname_from_path + ' in path') return False - base64_str = \ - auth_header.split(' ')[1].replace('\n', '').replace('\r', '') + base64_str1 = auth_header.split(' ')[1] + base64_str = remove_eol(base64_str1) plain = base64.b64decode(base64_str).decode('utf-8') if ':' not in plain: if debug: @@ -148,8 +149,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str, for line in passfile: if not line.startswith(nickname + ':'): continue - stored_password = \ - line.split(':')[1].replace('\n', '').replace('\r', '') + stored_password_base = line.split(':')[1] + stored_password = remove_eol(stored_password_base) success = _verify_password(stored_password, provided_password) if not success: if debug: @@ -169,8 +170,8 @@ def store_basic_credentials(base_dir: str, """ if ':' in nickname or ':' in password: return False - nickname = nickname.replace('\n', '').replace('\r', '').strip() - password = password.replace('\n', '').replace('\r', '').strip() + nickname = remove_eol(nickname).strip() + password = remove_eol(password).strip() if not os.path.isdir(base_dir + '/accounts'): os.mkdir(base_dir + '/accounts') diff --git a/blocking.py b/blocking.py index a9296afd5..625bc94c1 100644 --- a/blocking.py +++ b/blocking.py @@ -11,6 +11,7 @@ import os import json import time from datetime import datetime +from utils import remove_eol from utils import has_object_string from utils import has_object_string_object from utils import has_object_string_type @@ -162,8 +163,7 @@ def remove_global_block(base_dir: str, with open(unblocking_filename + '.new', 'w+', encoding='utf-8') as fpnew: for line in fp_unblock: - handle = \ - line.replace('\n', '').replace('\r', '') + handle = remove_eol(line) if unblock_handle not in line: fpnew.write(handle + '\n') except OSError as ex: @@ -189,8 +189,7 @@ def remove_global_block(base_dir: str, with open(unblocking_filename + '.new', 'w+', encoding='utf-8') as fpnew: for line in fp_unblock: - block_line = \ - line.replace('\n', '').replace('\r', '') + block_line = remove_eol(line) if unblock_hashtag not in line: fpnew.write(block_line + '\n') except OSError as ex: @@ -225,7 +224,7 @@ def remove_block(base_dir: str, nickname: str, domain: str, with open(unblocking_filename + '.new', 'w+', encoding='utf-8') as fpnew: for line in fp_unblock: - handle = line.replace('\n', '').replace('\r', '') + handle = remove_eol(line) if unblock_handle not in line: fpnew.write(handle + '\n') except OSError as ex: @@ -304,7 +303,7 @@ def update_blocked_cache(base_dir: str, blocked_lines = fp_blocked.readlines() # remove newlines for index, _ in enumerate(blocked_lines): - blocked_lines[index] = blocked_lines[index].replace('\n', '') + blocked_lines[index] = remove_eol(blocked_lines[index]) # update the cache blocked_cache.clear() blocked_cache += blocked_lines @@ -946,7 +945,7 @@ def set_broch_mode(base_dir: str, domain_full: str, enabled: bool) -> None: for handle in follow_list: if '@' not in handle: continue - handle = handle.replace('\n', '') + handle = remove_eol(handle) handle_domain = handle.split('@')[1] if handle_domain not in allowed_domains: allowed_domains.append(handle_domain) diff --git a/blog.py b/blog.py index 0d7a3e3cf..eb8766076 100644 --- a/blog.py +++ b/blog.py @@ -17,6 +17,7 @@ from webapp_utils import html_footer from webapp_utils import get_post_attachments_as_html from webapp_utils import edit_text_area from webapp_media import add_embedded_elements +from utils import remove_eol from utils import text_in_file from utils import local_actor_url from utils import get_actor_languages_list @@ -80,7 +81,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, print('EX: failed to read blog ' + post_filename) for reply_post_id in lines: - reply_post_id = reply_post_id.replace('\n', '').replace('\r', '') + reply_post_id = remove_eol(reply_post_id) reply_post_id = reply_post_id.replace('.json', '') if locate_post(base_dir, nickname, domain, reply_post_id): reply_post_id = reply_post_id.replace('.replies', '') @@ -99,8 +100,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {}, try: with open(post_filename, 'w+', encoding='utf-8') as post_file: for reply_post_id in lines: - reply_post_id = \ - reply_post_id.replace('\n', '').replace('\r', '') + reply_post_id = remove_eol(reply_post_id) if reply_post_id not in removals: post_file.write(reply_post_id + '\n') except OSError as ex: @@ -158,7 +158,7 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {}, if lines: replies_str = '' for reply_post_id in lines: - reply_post_id = reply_post_id.replace('\n', '').replace('\r', '') + reply_post_id = remove_eol(reply_post_id) reply_post_id = reply_post_id.replace('.json', '') reply_post_id = reply_post_id.replace('.replies', '') post_filename = acct_dir(base_dir, nickname, domain) + \ diff --git a/bookmarks.py b/bookmarks.py index dbb31116f..b14453d90 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -29,6 +29,7 @@ from utils import local_actor_url from utils import has_actor from utils import has_object_string_type from utils import text_in_file +from utils import remove_eol from posts import get_person_box from session import post_json @@ -71,7 +72,7 @@ def undo_bookmarks_collection_entry(recent_posts_cache: {}, bookmark_index = post_filename.split('/')[-1].strip() else: bookmark_index = post_filename.strip() - bookmark_index = bookmark_index.replace('\n', '').replace('\r', '') + bookmark_index = remove_eol(bookmark_index) if not text_in_file(bookmark_index, bookmarks_index_filename): return index_str = '' diff --git a/city.py b/city.py index 512418e5d..88867c8cb 100644 --- a/city.py +++ b/city.py @@ -13,6 +13,7 @@ import random import math from random import randint from utils import acct_dir +from utils import remove_eol # states which the simulated city dweller can be in PERSON_SLEEP = 0 @@ -145,7 +146,7 @@ def _get_city_pulse(curr_time_of_day, decoy_seed: int) -> (float, float): def parse_nogo_string(nogo_line: str) -> []: """Parses a line from locations_nogo.txt and returns the polygon """ - nogo_line = nogo_line.replace('\n', '').replace('\r', '') + nogo_line = remove_eol(nogo_line) polygon_str = nogo_line.split(':', 1)[1] if ';' in polygon_str: pts = polygon_str.split(';') @@ -320,7 +321,8 @@ def get_spoofed_city(city: str, base_dir: str, if os.path.isfile(city_filename): try: with open(city_filename, 'r', encoding='utf-8') as city_file: - city = city_file.read().replace('\n', '') + city1 = city_file.read() + city = remove_eol(city1) except OSError: print('EX: unable to read ' + city_filename) return city diff --git a/content.py b/content.py index adfca5b48..e5b1af646 100644 --- a/content.py +++ b/content.py @@ -32,6 +32,7 @@ from utils import acct_dir from utils import is_float from utils import get_currencies from utils import remove_html +from utils import remove_eol from petnames import get_pet_name from session import download_image @@ -288,7 +289,7 @@ def switch_words(base_dir: str, nickname: str, domain: str, content: str, print('EX: unable to read switches ' + switch_words_filename) for line in rules: - replace_str = line.replace('\n', '').replace('\r', '') + replace_str = remove_eol(line) splitters = ('->', ':', ',', ';', '-') word_transform = None for split_str in splitters: @@ -397,7 +398,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None: line = count_str + ' ' + emoji_content new_common_emoji.append(line) else: - new_common_emoji.append(line.replace('\n', '')) + line1 = remove_eol(line) + new_common_emoji.append(line1) if not emoji_found: new_common_emoji.append(str(1).zfill(16) + ' ' + emoji_content) new_common_emoji.sort(reverse=True) @@ -757,7 +759,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str, continue follow_nick = follow.split('@')[0] if possible_nickname == follow_nick: - follow_str = follow.replace('\n', '').replace('\r', '') + follow_str = remove_eol(follow) replace_domain = follow_str.split('@')[1] recipient_actor = http_prefix + "://" + \ replace_domain + "/@" + possible_nickname @@ -780,10 +782,10 @@ def _add_mention(word_str: str, http_prefix: str, following: str, if '@' not in follow: follow_ctr += 1 continue - pet = petnames[follow_ctr].replace('\n', '') + pet = remove_eol(petnames[follow_ctr]) if pet: if possible_nickname == pet: - follow_str = follow.replace('\n', '').replace('\r', '') + follow_str = remove_eol(follow) replace_nickname = follow_str.split('@')[0] replace_domain = follow_str.split('@')[1] recipient_actor = http_prefix + "://" + \ @@ -817,7 +819,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str, return False if following: for follow in following: - if follow.replace('\n', '').replace('\r', '') != possible_handle: + if remove_eol(follow) != possible_handle: continue recipient_actor = http_prefix + "://" + \ possible_domain + "/@" + possible_nickname diff --git a/crawlers.py b/crawlers.py index 1c9bc72b5..d95aa2aa4 100644 --- a/crawlers.py +++ b/crawlers.py @@ -11,6 +11,7 @@ import os import time from utils import save_json from utils import user_agent_domain +from utils import remove_eol from blocking import update_blocked_cache from blocking import is_blocked_domain @@ -72,7 +73,7 @@ def load_known_web_bots(base_dir: str) -> []: for crawler in crawlers_list: if not crawler: continue - crawler = crawler.replace('\n', '').strip() + crawler = remove_eol(crawler).strip() if not crawler: continue if crawler not in known_bots: diff --git a/daemon.py b/daemon.py index 521d13616..9eaf687bb 100644 --- a/daemon.py +++ b/daemon.py @@ -253,6 +253,7 @@ from languages import set_actor_languages from languages import get_understood_languages from like import update_likes_collection from reaction import update_reaction_collection +from utils import remove_eol from utils import text_in_file from utils import is_onion_request from utils import is_i2p_request @@ -4185,7 +4186,7 @@ class PubServer(BaseHTTPRequestHandler): timezone = \ self.server.account_timezone.get(nickname) - profile_handle = search_str.replace('\n', '').strip() + profile_handle = remove_eol(search_str).strip() # establish the session curr_proxy_type = proxy_type diff --git a/epicyon.py b/epicyon.py index d9b13445f..f55158717 100644 --- a/epicyon.py +++ b/epicyon.py @@ -68,6 +68,7 @@ from tests import test_update_actor from tests import run_all_tests from auth import store_basic_credentials from auth import create_password +from utils import remove_eol from utils import text_in_file from utils import remove_domain_port from utils import get_port_from_domain @@ -1457,7 +1458,8 @@ def _command_options() -> None: with open(approve_follows_filename, 'r', encoding='utf-8') as approvefile: for approve in approvefile: - print(approve.replace('\n', '').replace('\r', '')) + approve1 = remove_eol(approve) + print(approve1) approve_ctr += 1 if approve_ctr == 0: print('There are no follow requests pending approval.') @@ -1488,7 +1490,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) if not argb.sendto: @@ -1504,7 +1506,7 @@ def _command_options() -> None: if '@' in argb.sendto: to_nickname = argb.sendto.split('@')[0] to_domain = argb.sendto.split('@')[1] - to_domain = to_domain.replace('\n', '').replace('\r', '') + to_domain = remove_eol(to_domain) to_port = 443 if ':' in to_domain: to_port = get_port_from_domain(to_domain) @@ -1583,7 +1585,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) proxy_type = None if argb.tor or domain.endswith('.onion'): proxy_type = 'tor' @@ -1624,7 +1626,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -1661,7 +1663,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) proxy_type = None if argb.tor or domain.endswith('.onion'): @@ -1697,7 +1699,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if not argb.nickname: print('Specify a nickname with the --nickname option') @@ -1769,7 +1771,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if not argb.nickname: print('Specify a nickname with the --nickname option') @@ -1803,7 +1805,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if not argb.nickname: print('Specify a nickname with the --nickname option') @@ -1875,7 +1877,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if not argb.nickname: print('Specify a nickname with the --nickname option') @@ -1913,7 +1915,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -1952,7 +1954,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -1985,7 +1987,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2025,7 +2027,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2060,7 +2062,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2094,7 +2096,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2127,7 +2129,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2164,7 +2166,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) follow_nickname = get_nickname_from_actor(argb.follow) if not follow_nickname: @@ -2212,7 +2214,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) follow_nickname = get_nickname_from_actor(argb.unfollow) if not follow_nickname: @@ -2257,7 +2259,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2290,7 +2292,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2324,7 +2326,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2442,31 +2444,31 @@ def _command_options() -> None: sys.exit() if '/users/' in argb.followers: nickname = argb.followers.split('/users/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/users/')[0] elif '/profile/' in argb.followers: nickname = argb.followers.split('/profile/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/profile/')[0] elif '/author/' in argb.followers: nickname = argb.followers.split('/author/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/author/')[0] elif '/channel/' in argb.followers: nickname = argb.followers.split('/channel/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/channel/')[0] elif '/accounts/' in argb.followers: nickname = argb.followers.split('/accounts/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/accounts/')[0] elif '/u/' in argb.followers: nickname = argb.followers.split('/u/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/u/')[0] elif '/c/' in argb.followers: nickname = argb.followers.split('/c/')[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = argb.followers.split('/c/')[0] else: # format: @nick@domain @@ -2480,7 +2482,7 @@ def _command_options() -> None: sys.exit() nickname = argb.followers.split('@')[0] domain = argb.followers.split('@')[1] - domain = domain.replace('\n', '').replace('\r', '') + domain = remove_eol(domain) cached_webfingers = {} if argb.http or domain.endswith('.onion'): http_prefix = 'http' @@ -2603,7 +2605,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if len(argb.password.strip()) < 8: print('Password should be at least 8 characters') sys.exit() @@ -2652,7 +2654,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if len(argb.password.strip()) < 8: print('Password should be at least 8 characters') sys.exit() @@ -2818,7 +2820,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if not argb.skillLevelPercent: print('Specify a skill level in the range 0-100') @@ -2862,7 +2864,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -2906,7 +2908,7 @@ def _command_options() -> None: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) proxy_type = None if argb.tor or domain.endswith('.onion'): @@ -2965,11 +2967,11 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if '@' in argb.block: blocked_domain = argb.block.split('@')[1] - blocked_domain = blocked_domain.replace('\n', '').replace('\r', '') + blocked_domain = remove_eol(blocked_domain) blocked_nickname = argb.block.split('@')[0] blocked_actor = http_prefix + '://' + blocked_domain + \ '/users/' + blocked_nickname @@ -3009,7 +3011,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -3041,7 +3043,7 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) session = create_session(proxy_type) person_cache = {} @@ -3073,11 +3075,11 @@ def _command_options() -> None: if not argb.password: print('Specify a password with the --password option') sys.exit() - argb.password = argb.password.replace('\n', '') + argb.password = remove_eol(argb.password) if '@' in argb.unblock: blocked_domain = argb.unblock.split('@')[1] - blocked_domain = blocked_domain.replace('\n', '').replace('\r', '') + blocked_domain = remove_eol(blocked_domain) blocked_nickname = argb.unblock.split('@')[0] blocked_actor = http_prefix + '://' + blocked_domain + \ '/users/' + blocked_nickname diff --git a/filters.py b/filters.py index 14c95b2bd..329528ac6 100644 --- a/filters.py +++ b/filters.py @@ -10,6 +10,7 @@ __module_group__ = "Moderation" import os from utils import acct_dir from utils import text_in_file +from utils import remove_eol def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool: @@ -62,7 +63,7 @@ def remove_filter(base_dir: str, nickname: str, domain: str, with open(filters_filename, 'r', encoding='utf-8') as fp_filt: with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: for line in fp_filt: - line = line.replace('\n', '') + line = remove_eol(line) if line != words: fpnew.write(line + '\n') except OSError as ex: @@ -87,7 +88,7 @@ def remove_global_filter(base_dir: str, words: str) -> bool: with open(filters_filename, 'r', encoding='utf-8') as fp_filt: with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: for line in fp_filt: - line = line.replace('\n', '') + line = remove_eol(line) if line != words: fpnew.write(line + '\n') except OSError as ex: @@ -122,7 +123,7 @@ def _is_filtered_base(filename: str, content: str) -> bool: try: with open(filename, 'r', encoding='utf-8') as fp_filt: for line in fp_filt: - filter_str = line.replace('\n', '').replace('\r', '') + filter_str = remove_eol(line) if not filter_str: continue if len(filter_str) < 2: diff --git a/follow.py b/follow.py index a8573fe84..dedaa61e8 100644 --- a/follow.py +++ b/follow.py @@ -31,6 +31,7 @@ from utils import acct_dir from utils import has_group_type from utils import local_actor_url from utils import text_in_file +from utils import remove_eol from acceptreject import create_accept from acceptreject import create_reject from webfinger import webfinger_handle @@ -67,7 +68,7 @@ def create_initial_last_seen(base_dir: str, http_prefix: str) -> None: continue if '@' not in handle: continue - handle = handle.replace('\n', '') + handle = remove_eol(handle) nickname = handle.split('@')[0] domain = handle.split('@')[1] if nickname.startswith('!'): @@ -244,7 +245,7 @@ def get_follower_domains(base_dir: str, nickname: str, domain: str) -> []: domains_list = [] for handle in lines: - handle = handle.replace('\n', '') + handle = remove_eol(handle) follower_domain, _ = get_domain_from_actor(handle) if not follower_domain: continue @@ -535,8 +536,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str, page_ctr += 1 total_ctr += 1 if curr_page == page_number: - line2 = \ - line.lower().replace('\n', '').replace('\r', '') + line2_lower = line.lower() + line2 = remove_eol(line2_lower) nick = line2.split('@')[0] dom = line2.split('@')[1] if not nick.startswith('!'): @@ -555,8 +556,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str, page_ctr += 1 total_ctr += 1 if curr_page == page_number: - append_str = \ - line.lower().replace('\n', '').replace('\r', '') + append_str1 = line.lower() + append_str = remove_eol(append_str1) following['orderedItems'].append(append_str) if page_ctr >= follows_per_page: page_ctr = 0 diff --git a/happening.py b/happening.py index adb67df2d..87188c233 100644 --- a/happening.py +++ b/happening.py @@ -25,6 +25,7 @@ from utils import delete_post from utils import get_status_number from utils import get_full_domain from utils import text_in_file +from utils import remove_eol from filters import is_filtered from context import get_individual_post_context from session import get_method @@ -262,7 +263,7 @@ def get_todays_events(base_dir: str, nickname: str, domain: str, recreate_events_file = False with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: - post_id = post_id.replace('\n', '').replace('\r', '') + post_id = remove_eol(post_id) post_filename = locate_post(base_dir, nickname, domain, post_id) if not post_filename: recreate_events_file = True @@ -556,7 +557,7 @@ def day_events_check(base_dir: str, nickname: str, domain: str, events_exist = False with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: - post_id = post_id.replace('\n', '').replace('\r', '') + post_id = remove_eol(post_id) post_filename = locate_post(base_dir, nickname, domain, post_id) if not post_filename: continue @@ -612,7 +613,7 @@ def get_this_weeks_events(base_dir: str, nickname: str, domain: str) -> {}: recreate_events_file = False with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: - post_id = post_id.replace('\n', '').replace('\r', '') + post_id = remove_eol(post_id) post_filename = locate_post(base_dir, nickname, domain, post_id) if not post_filename: recreate_events_file = True @@ -679,7 +680,7 @@ def get_calendar_events(base_dir: str, nickname: str, domain: str, recreate_events_file = False with open(calendar_filename, 'r', encoding='utf-8') as events_file: for post_id in events_file: - post_id = post_id.replace('\n', '').replace('\r', '') + post_id = remove_eol(post_id) post_filename = locate_post(base_dir, nickname, domain, post_id) if not post_filename: recreate_events_file = True diff --git a/inbox.py b/inbox.py index e832b4641..8b64d4a1b 100644 --- a/inbox.py +++ b/inbox.py @@ -18,6 +18,7 @@ from languages import understood_post_language from like import update_likes_collection from reaction import update_reaction_collection from reaction import valid_emoji_content +from utils import remove_eol from utils import text_in_file from utils import get_media_descriptions_from_post from utils import get_summary_from_post @@ -2314,7 +2315,8 @@ def _receive_announce(recent_posts_cache: {}, str(message_json)) else: if debug: - print('Generated announce html ' + announce_html.replace('\n', '')) + announce_html2 = remove_eol(announce_html) + print('Generated announce html ' + announce_html2) post_json_object = download_announce(session, base_dir, http_prefix, diff --git a/manualapprove.py b/manualapprove.py index 734547e0d..da9710e9b 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -17,6 +17,7 @@ from utils import get_port_from_domain from utils import get_user_paths from utils import acct_dir from utils import text_in_file +from utils import remove_eol from threads import thread_with_trace from session import create_session @@ -57,8 +58,7 @@ def manual_deny_follow_request(session, session_onion, session_i2p, print('EX: unable to append ' + rejected_follows_filename) deny_nickname = deny_handle.split('@')[0] - deny_domain = \ - deny_handle.split('@')[1].replace('\n', '').replace('\r', '') + deny_domain = remove_eol(deny_handle.split('@')[1]) deny_port = port if ':' in deny_domain: deny_port = get_port_from_domain(deny_domain) @@ -195,7 +195,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p, # is this the approved follow? if handle_of_follow_requester.startswith(approve_handle_full): handle_of_follow_requester = \ - handle_of_follow_requester.replace('\n', '') + remove_eol(handle_of_follow_requester) handle_of_follow_requester = \ handle_of_follow_requester.replace('\r', '') port2 = port @@ -212,7 +212,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p, approve_nickname = approve_handle.split('@')[0] approve_domain = approve_handle.split('@')[1] approve_domain = \ - approve_domain.replace('\n', '') + remove_eol(approve_domain) approve_domain = \ approve_domain.replace('\r', '') approve_port = port2 diff --git a/newswire.py b/newswire.py index f0360dcd9..ed278a7e7 100644 --- a/newswire.py +++ b/newswire.py @@ -20,6 +20,7 @@ from datetime import timezone from collections import OrderedDict from utils import valid_post_date from categories import set_hashtag_category +from utils import remove_eol from utils import get_domain_from_actor from utils import valid_hash_tag from utils import dangerous_svg @@ -1458,8 +1459,7 @@ def _add_account_blogs_to_newswire(base_dir: str, nickname: str, domain: str, # filename of the post without any extension or path # This should also correspond to any index entry in # the posts cache - post_url = \ - post_filename.replace('\n', '').replace('\r', '') + post_url = remove_eol(post_filename) post_url = post_url.replace('.json', '').strip() # read the post from file diff --git a/person.py b/person.py index a24c85cc3..c8a1d8679 100644 --- a/person.py +++ b/person.py @@ -43,7 +43,7 @@ from utils import get_nickname_from_actor from utils import remove_html from utils import contains_invalid_chars from utils import replace_users_with_at -from utils import remove_line_endings +from utils import remove_eol from utils import remove_domain_port from utils import get_status_number from utils import get_full_domain @@ -63,6 +63,7 @@ from utils import get_group_paths from utils import local_actor_url from utils import dangerous_svg from utils import text_in_file +from utils import remove_line_endings from session import create_session from session import get_json from webfinger import webfinger_handle @@ -104,7 +105,7 @@ def set_profile_image(base_dir: str, http_prefix: str, """Saves the given image file as an avatar or background image for the given person """ - image_filename = image_filename.replace('\n', '').replace('\r', '') + image_filename = remove_eol(image_filename) if not is_image_file(image_filename): print('Profile image must be png, jpg, gif or svg format') return False @@ -1371,8 +1372,8 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str, for line in snoozed_file: # is this the entry for the actor? if line.startswith(snooze_actor + ' '): - snoozed_time_str = \ - line.split(' ')[1].replace('\n', '').replace('\r', '') + snoozed_time_str1 = line.split(' ')[1] + snoozed_time_str = remove_eol(snoozed_time_str1) # is there a time appended? if snoozed_time_str.isdigit(): snoozed_time = int(snoozed_time_str) @@ -1525,7 +1526,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool, for user_path in paths: if user_path in handle: nickname = handle.split(user_path)[1] - nickname = nickname.replace('\n', '').replace('\r', '') + nickname = remove_eol(nickname) domain = handle.split(user_path)[0] user_path_found = True break @@ -1556,7 +1557,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool, return None, None nickname = handle.split('@')[0] domain = handle.split('@')[1] - domain = domain.replace('\n', '').replace('\r', '') + domain = remove_eol(domain) cached_webfingers = {} proxy_type = None diff --git a/posts.py b/posts.py index 19060b1ba..8a1d07fd1 100644 --- a/posts.py +++ b/posts.py @@ -32,6 +32,7 @@ from webfinger import webfinger_handle from httpsig import create_signed_header from siteactive import site_is_active from languages import understood_post_language +from utils import remove_eol from utils import text_in_file from utils import get_media_descriptions_from_post from utils import valid_hash_tag @@ -139,8 +140,7 @@ def no_of_followers_on_domain(base_dir: str, handle: str, for follower_handle in followers_file: if '@' in follower_handle: follower_domain = follower_handle.split('@')[1] - follower_domain = follower_domain.replace('\n', '') - follower_domain = follower_domain.replace('\r', '') + follower_domain = remove_eol(follower_domain) if domain == follower_domain: ctr += 1 return ctr @@ -2707,8 +2707,8 @@ def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}: for follower_handle in foll_file: if '@' not in follower_handle: continue - fhandle = \ - follower_handle.strip().replace('\n', '').replace('\r', '') + fhandle1 = follower_handle.strip() + fhandle = remove_eol(fhandle1) follower_domain = fhandle.split('@')[1] if not grouped.get(follower_domain): grouped[follower_domain] = [fhandle] @@ -4041,8 +4041,7 @@ def _create_box_indexed(recent_posts_cache: {}, # filename of the post without any extension or path # This should also correspond to any index entry in # the posts cache - post_url = \ - post_filename.replace('\n', '').replace('\r', '') + post_url = remove_eol(post_filename) post_url = post_url.replace('.json', '').strip() if post_url in post_urls_in_box: @@ -4737,7 +4736,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str, reply_found = False # examine inbox and outbox for boxname in replies_boxes: - message_id2 = message_id.replace('\n', '').replace('\r', '') + message_id2 = remove_eol(message_id) search_filename = \ acct_dir(base_dir, nickname, domain) + '/' + \ boxname + '/' + \ @@ -4763,7 +4762,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str, break # if not in either inbox or outbox then examine the shared inbox if not reply_found: - message_id2 = message_id.replace('\n', '').replace('\r', '') + message_id2 = remove_eol(message_id) search_filename = \ base_dir + \ '/accounts/inbox@' + \ diff --git a/reaction.py b/reaction.py index 74ca6dc9e..f3f618382 100644 --- a/reaction.py +++ b/reaction.py @@ -31,6 +31,7 @@ from utils import save_json from utils import remove_post_from_cache from utils import get_cached_post_filename from utils import contains_invalid_chars +from utils import remove_eol from posts import send_signed_json from session import post_json from webfinger import webfinger_handle @@ -470,7 +471,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None: line = count_str + ' ' + emoji_content new_common_reactions.append(line) else: - new_common_reactions.append(line.replace('\n', '')) + line1 = remove_eol(line) + new_common_reactions.append(line1) if not reaction_found: new_common_reactions.append(str(1).zfill(16) + ' ' + emoji_content) new_common_reactions.sort(reverse=True) diff --git a/schedule.py b/schedule.py index d8fa45576..52460847b 100644 --- a/schedule.py +++ b/schedule.py @@ -15,6 +15,7 @@ from utils import get_status_number from utils import load_json from utils import is_account_dir from utils import acct_dir +from utils import remove_eol from outbox import post_message_to_outbox from session import create_session @@ -43,7 +44,8 @@ def _update_post_schedule(base_dir: str, handle: str, httpd, date_str = line.split(' ')[0] if 'T' not in date_str: continue - post_id = line.split(' ', 1)[1].replace('\n', '').replace('\r', '') + post_id1 = line.split(' ', 1)[1] + post_id = remove_eol(post_id1) post_filename = schedule_dir + post_id + '.json' if delete_schedule_post: # delete extraneous scheduled posts diff --git a/shares.py b/shares.py index 850039c8b..2d9bad60e 100644 --- a/shares.py +++ b/shares.py @@ -22,6 +22,7 @@ from posts import get_person_box from session import post_json from session import post_image from session import create_session +from utils import remove_eol from utils import has_object_string_type from utils import date_string_to_seconds from utils import date_seconds_to_string @@ -1502,7 +1503,7 @@ def authorize_shared_items(shared_items_federated_domains: [], if debug: print('DEBUG: shared item federation should not use basic auth') return False - provided_token = auth_header.replace('\n', '').replace('\r', '').strip() + provided_token = remove_eol(auth_header).strip() if not provided_token: if debug: print('DEBUG: shared item federation token is empty') diff --git a/tests.py b/tests.py index a635cd622..31f7b0746 100644 --- a/tests.py +++ b/tests.py @@ -54,6 +54,7 @@ from follow import clear_followers from follow import send_follow_request_via_server from follow import send_unfollow_request_via_server from siteactive import site_is_active +from utils import remove_eol from utils import text_in_file from utils import convert_published_to_local_timezone from utils import convert_to_snake_case @@ -2741,7 +2742,7 @@ def _test_follows(base_dir: str) -> None: domain_found = False for following_domain in fp_foll: test_domain = following_domain.split('@')[1] - test_domain = test_domain.replace('\n', '').replace('\r', '') + test_domain = remove_eol(test_domain) if test_domain == 'mesh.com': domain_found = True if test_domain not in federation_list: @@ -2755,7 +2756,7 @@ def _test_follows(base_dir: str) -> None: domain_found = False for following_domain in fp_foll: test_domain = following_domain.split('@')[1] - test_domain = test_domain.replace('\n', '').replace('\r', '') + test_domain = remove_eol(test_domain) if test_domain == 'mesh.com': domain_found = True assert domain_found is False @@ -2779,7 +2780,7 @@ def _test_follows(base_dir: str) -> None: encoding='utf-8') as fp_foll: for follower_domain in fp_foll: test_domain = follower_domain.split('@')[1] - test_domain = test_domain.replace('\n', '').replace('\r', '') + test_domain = remove_eol(test_domain) if test_domain not in federation_list: print(test_domain) assert False @@ -4715,7 +4716,7 @@ def _get_function_call_args(name: str, lines: [], start_line_ctr: int) -> []: continue args_str += lines[line_ctr].split(')')[0] break - return args_str.replace('\n', '').replace(' ', '').split(',') + return remove_eol(args_str).replace(' ', '').split(',') def get_function_calls(name: str, lines: [], start_line_ctr: int, @@ -7257,6 +7258,15 @@ def _test_color_contrast_value(base_dir: str) -> None: print('Color contrast is ok for all themes') +def _test_remove_end_of_line(): + print('remove_end_of_line') + text = 'some text\r\n' + expected = 'some text' + assert remove_eol(text) == expected + text = 'some text' + assert remove_eol(text) == expected + + def run_all_tests(): base_dir = os.getcwd() print('Running tests...') @@ -7274,6 +7284,7 @@ def run_all_tests(): _test_checkbox_names() _test_thread_functions() _test_functions() + _test_remove_end_of_line() _test_translation_labels() _test_color_contrast_value(base_dir) _test_diff_content() diff --git a/theme.py b/theme.py index a17ca70f1..3151058b7 100644 --- a/theme.py +++ b/theme.py @@ -18,6 +18,7 @@ from utils import dangerous_svg from utils import local_actor_url from utils import remove_html from utils import text_in_file +from utils import remove_eol from shutil import copyfile from shutil import make_archive from shutil import unpack_archive @@ -44,7 +45,8 @@ def import_theme(base_dir: str, filename: str) -> bool: new_theme_name = None with open(temp_theme_dir + '/name.txt', 'r', encoding='utf-8') as fp_theme: - new_theme_name = fp_theme.read().replace('\n', '').replace('\r', '') + new_theme_name1 = fp_theme.read() + new_theme_name = remove_eol(new_theme_name1) if len(new_theme_name) > 20: print('WARN: Imported theme name is too long') return False diff --git a/utils.py b/utils.py index 2da0d2e30..6cea9162c 100644 --- a/utils.py +++ b/utils.py @@ -40,6 +40,12 @@ INVALID_CHARACTERS = ( ) +def remove_eol(line: str): + """Removes line ending characters + """ + return line.replace('\n', '').replace('\r', '') + + def text_in_file(text: str, filename: str, case_sensitive: bool = True) -> bool: """is the given text in the given file? @@ -671,8 +677,7 @@ def get_followers_of_person(base_dir: str, continue with open(filename, 'r', encoding='utf-8') as followingfile: for following_handle in followingfile: - following_handle2 = following_handle.replace('\n', '') - following_handle2 = following_handle2.replace('\r', '') + following_handle2 = remove_eol(following_handle) if following_handle2 == handle: if account not in followers: followers.append(account) @@ -1306,7 +1311,8 @@ def follow_person(base_dir: str, nickname: str, domain: str, follow_file: str = 'following.txt') -> bool: """Adds a person to the follow list """ - follow_domain_str_lower = follow_domain.lower().replace('\n', '') + follow_domain_str_lower1 = follow_domain.lower() + follow_domain_str_lower = remove_eol(follow_domain_str_lower1) if not domain_permitted(follow_domain_str_lower, federation_list): if debug: @@ -1414,8 +1420,8 @@ def locate_news_votes(base_dir: str, domain: str, """Returns the votes filename for a news post within the news user account """ - post_url = \ - post_url.strip().replace('\n', '').replace('\r', '') + post_url1 = post_url.strip() + post_url = remove_eol(post_url1) # if this post in the shared inbox? post_url = remove_id_ending(post_url.strip()).replace('/', '#') @@ -1438,8 +1444,8 @@ def locate_news_arrival(base_dir: str, domain: str, """Returns the arrival time for a news post within the news user account """ - post_url = \ - post_url.strip().replace('\n', '').replace('\r', '') + post_url1 = post_url.strip() + post_url = remove_eol(post_url1) # if this post in the shared inbox? post_url = remove_id_ending(post_url.strip()).replace('/', '#') @@ -2910,8 +2916,7 @@ def reject_post_id(base_dir: str, nickname: str, domain: str, # filename of the post without any extension or path # This should also correspond to any index entry in # the posts cache - post_url = \ - index_filename.replace('\n', '').replace('\r', '') + post_url = remove_eol(index_filename) post_url = post_url.replace('.json', '').strip() if post_url in recent_posts_cache['index']: diff --git a/webapp_moderation.py b/webapp_moderation.py index 180bd40f3..a3914ce7b 100644 --- a/webapp_moderation.py +++ b/webapp_moderation.py @@ -17,6 +17,7 @@ from utils import get_nickname_from_actor from utils import get_domain_from_actor from utils import get_config_param from utils import local_actor_url +from utils import remove_eol from posts import download_follow_collection from posts import get_public_post_info from posts import is_moderator @@ -418,7 +419,7 @@ def html_moderation_info(translate: {}, base_dir: str, for line in blocked_lines: if not line: continue - line = line.replace('\n', '').replace('\r', '').strip() + line = remove_eol(line).strip() blocked_str += line + '\n' info_form += '
\n' info_form += \ diff --git a/webapp_post.py b/webapp_post.py index 3c198622f..0aeb895b5 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -25,6 +25,7 @@ from posts import post_is_muted from posts import get_person_box from posts import download_announce from posts import populate_replies_json +from utils import remove_eol from utils import disallow_announce from utils import disallow_reply from utils import convert_published_to_local_timezone @@ -1542,7 +1543,8 @@ def _substitute_onion_domains(base_dir: str, content: str) -> str: if sep not in line: continue clearnet = line.split(sep, 1)[0].strip() - onion = line.split(sep, 1)[1].strip().replace('\n', '') + onion1 = line.split(sep, 1)[1].strip() + onion = remove_eol(onion1) if clearnet and onion: onion_domains[clearnet] = onion break diff --git a/webapp_profile.py b/webapp_profile.py index 59932f501..e024cb659 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -30,6 +30,7 @@ from utils import get_supported_languages from utils import local_actor_url from utils import get_reply_interval_hours from utils import get_account_timezone +from utils import remove_eol from languages import get_actor_languages from skills import get_skills from theme import get_themes_list @@ -796,7 +797,8 @@ def html_profile(signing_priv_key_pem: str, encoding='utf-8') as req_file: for follower_handle in req_file: if len(follower_handle) > 0: - follower_handle = follower_handle.replace('\n', '') + follower_handle = \ + remove_eol(follower_handle) if '://' in follower_handle: follower_actor = follower_handle else: @@ -1751,7 +1753,8 @@ def _html_edit_profile_filtering(base_dir: str, nickname: str, domain: str, city_filename = acct_dir(base_dir, nickname, domain) + '/city.txt' if os.path.isfile(city_filename): with open(city_filename, 'r', encoding='utf-8') as city_file: - city = city_file.read().replace('\n', '') + city1 = city_file.read() + city = remove_eol(city1) locations_filename = base_dir + '/custom_locations.txt' if not os.path.isfile(locations_filename): locations_filename = base_dir + '/locations.txt' diff --git a/webapp_timeline.py b/webapp_timeline.py index 29e13dce9..515893e1c 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -19,6 +19,7 @@ from utils import remove_id_ending from utils import acct_dir from utils import is_float from utils import local_actor_url +from utils import remove_eol from follow import follower_approval_active from person import is_person_snoozed from markdown import markdown_to_html @@ -514,7 +515,7 @@ def html_timeline(css_cache: {}, default_timeline: str, calendar_image = 'calendar_notify.png' with open(calendar_file, 'r', encoding='utf-8') as calfile: calendar_path = calfile.read().replace('##sent##', '') - calendar_path = calendar_path.replace('\n', '').replace('\r', '') + calendar_path = remove_eol(calendar_path) if '/calendar' not in calendar_path: calendar_path = '/calendar' diff --git a/webapp_utils.py b/webapp_utils.py index a0b45c2a7..1a53850ab 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -28,6 +28,7 @@ from utils import get_video_extensions from utils import get_image_extensions from utils import local_actor_url from utils import text_in_file +from utils import remove_eol from cache import store_person_in_cache from content import add_html_tags from content import replace_emoji_from_tags @@ -1822,7 +1823,8 @@ def html_common_emoji(base_dir: str, no_of_emoji: int) -> str: ctr = 0 html_str = '' while ctr < no_of_emoji and line_ctr < len(common_emoji): - emoji_name = common_emoji[line_ctr].split(' ')[1].replace('\n', '') + emoji_name1 = common_emoji[line_ctr].split(' ')[1] + emoji_name = remove_eol(emoji_name1) emoji_icon_name = emoji_name emoji_filename = base_dir + '/emoji/' + emoji_name + '.png' if not os.path.isfile(emoji_filename):