indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Merge branch 'main' of gitlab.com:bashrc2/epicyon

merge-requests/30/head
Bob Mottram 2022-06-21 14:35:20 +01:00
parent 4b92a03da0 65d9b0680a
commit b65908589d
28 changed files with 174 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
auth.py
View File

@ -16,6 +16,7 @@ import datetime
from utils import is_system_account from utils import is_system_account
from utils import has_users_path from utils import has_users_path
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
def _hash_password(password: str) -> str: def _hash_password(password: str) -> str:
@ -78,9 +79,9 @@ def create_basic_auth_header(nickname: str, password: str) -> str:
"""This is only used by tests """This is only used by tests
""" """
auth_str = \ auth_str = \
nickname.replace('\n', '').replace('\r', '') + \ remove_eol(nickname) + \
':' + \ ':' + \
password.replace('\n', '').replace('\r', '') remove_eol(password)
return 'Basic ' + \ return 'Basic ' + \
base64.b64encode(auth_str.encode('utf-8')).decode('utf-8') base64.b64encode(auth_str.encode('utf-8')).decode('utf-8')
@ -118,8 +119,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str,
print('basic auth - attempted login using system account ' + print('basic auth - attempted login using system account ' +
nickname_from_path + ' in path') nickname_from_path + ' in path')
return False return False
base64_str = \ base64_str1 = auth_header.split(' ')[1]
auth_header.split(' ')[1].replace('\n', '').replace('\r', '') base64_str = remove_eol(base64_str1)
plain = base64.b64decode(base64_str).decode('utf-8') plain = base64.b64decode(base64_str).decode('utf-8')
if ':' not in plain: if ':' not in plain:
if debug: if debug:
@ -148,8 +149,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str,
for line in passfile: for line in passfile:
if not line.startswith(nickname + ':'): if not line.startswith(nickname + ':'):
continue continue
stored_password = \ stored_password_base = line.split(':')[1]
line.split(':')[1].replace('\n', '').replace('\r', '') stored_password = remove_eol(stored_password_base)
success = _verify_password(stored_password, provided_password) success = _verify_password(stored_password, provided_password)
if not success: if not success:
if debug: if debug:
@ -169,8 +170,8 @@ def store_basic_credentials(base_dir: str,
""" """
if ':' in nickname or ':' in password: if ':' in nickname or ':' in password:
return False return False
nickname = nickname.replace('\n', '').replace('\r', '').strip() nickname = remove_eol(nickname).strip()
password = password.replace('\n', '').replace('\r', '').strip() password = remove_eol(password).strip()
if not os.path.isdir(base_dir + '/accounts'): if not os.path.isdir(base_dir + '/accounts'):
os.mkdir(base_dir + '/accounts') os.mkdir(base_dir + '/accounts')

13
blocking.py
View File

@ -11,6 +11,7 @@ import os
import json import json
import time import time
from datetime import datetime from datetime import datetime
from utils import remove_eol
from utils import has_object_string from utils import has_object_string
from utils import has_object_string_object from utils import has_object_string_object
from utils import has_object_string_type from utils import has_object_string_type
@ -162,8 +163,7 @@ def remove_global_block(base_dir: str,
with open(unblocking_filename + '.new', 'w+', with open(unblocking_filename + '.new', 'w+',
encoding='utf-8') as fpnew: encoding='utf-8') as fpnew:
for line in fp_unblock: for line in fp_unblock:
handle = \ handle = remove_eol(line)
line.replace('\n', '').replace('\r', '')
if unblock_handle not in line: if unblock_handle not in line:
fpnew.write(handle + '\n') fpnew.write(handle + '\n')
except OSError as ex: except OSError as ex:
@ -189,8 +189,7 @@ def remove_global_block(base_dir: str,
with open(unblocking_filename + '.new', 'w+', with open(unblocking_filename + '.new', 'w+',
encoding='utf-8') as fpnew: encoding='utf-8') as fpnew:
for line in fp_unblock: for line in fp_unblock:
block_line = \ block_line = remove_eol(line)
line.replace('\n', '').replace('\r', '')
if unblock_hashtag not in line: if unblock_hashtag not in line:
fpnew.write(block_line + '\n') fpnew.write(block_line + '\n')
except OSError as ex: except OSError as ex:
@ -225,7 +224,7 @@ def remove_block(base_dir: str, nickname: str, domain: str,
with open(unblocking_filename + '.new', 'w+', with open(unblocking_filename + '.new', 'w+',
encoding='utf-8') as fpnew: encoding='utf-8') as fpnew:
for line in fp_unblock: for line in fp_unblock:
handle = line.replace('\n', '').replace('\r', '') handle = remove_eol(line)
if unblock_handle not in line: if unblock_handle not in line:
fpnew.write(handle + '\n') fpnew.write(handle + '\n')
except OSError as ex: except OSError as ex:
@ -304,7 +303,7 @@ def update_blocked_cache(base_dir: str,
blocked_lines = fp_blocked.readlines() blocked_lines = fp_blocked.readlines()
# remove newlines # remove newlines
for index, _ in enumerate(blocked_lines): for index, _ in enumerate(blocked_lines):
blocked_lines[index] = blocked_lines[index].replace('\n', '') blocked_lines[index] = remove_eol(blocked_lines[index])
# update the cache # update the cache
blocked_cache.clear() blocked_cache.clear()
blocked_cache += blocked_lines blocked_cache += blocked_lines
@ -946,7 +945,7 @@ def set_broch_mode(base_dir: str, domain_full: str, enabled: bool) -> None:
for handle in follow_list: for handle in follow_list:
if '@' not in handle: if '@' not in handle:
continue continue
handle = handle.replace('\n', '') handle = remove_eol(handle)
handle_domain = handle.split('@')[1] handle_domain = handle.split('@')[1]
if handle_domain not in allowed_domains: if handle_domain not in allowed_domains:
allowed_domains.append(handle_domain) allowed_domains.append(handle_domain)

8
blog.py
View File

@ -17,6 +17,7 @@ from webapp_utils import html_footer
from webapp_utils import get_post_attachments_as_html from webapp_utils import get_post_attachments_as_html
from webapp_utils import edit_text_area from webapp_utils import edit_text_area
from webapp_media import add_embedded_elements from webapp_media import add_embedded_elements
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import local_actor_url from utils import local_actor_url
from utils import get_actor_languages_list from utils import get_actor_languages_list
@ -80,7 +81,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {},
print('EX: failed to read blog ' + post_filename) print('EX: failed to read blog ' + post_filename)
for reply_post_id in lines: for reply_post_id in lines:
reply_post_id = reply_post_id.replace('\n', '').replace('\r', '') reply_post_id = remove_eol(reply_post_id)
reply_post_id = reply_post_id.replace('.json', '') reply_post_id = reply_post_id.replace('.json', '')
if locate_post(base_dir, nickname, domain, reply_post_id): if locate_post(base_dir, nickname, domain, reply_post_id):
reply_post_id = reply_post_id.replace('.replies', '') reply_post_id = reply_post_id.replace('.replies', '')
@ -99,8 +100,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {},
try: try:
with open(post_filename, 'w+', encoding='utf-8') as post_file: with open(post_filename, 'w+', encoding='utf-8') as post_file:
for reply_post_id in lines: for reply_post_id in lines:
reply_post_id = \ reply_post_id = remove_eol(reply_post_id)
reply_post_id.replace('\n', '').replace('\r', '')
if reply_post_id not in removals: if reply_post_id not in removals:
post_file.write(reply_post_id + '\n') post_file.write(reply_post_id + '\n')
except OSError as ex: except OSError as ex:
@ -158,7 +158,7 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {},
if lines: if lines:
replies_str = '' replies_str = ''
for reply_post_id in lines: for reply_post_id in lines:
reply_post_id = reply_post_id.replace('\n', '').replace('\r', '') reply_post_id = remove_eol(reply_post_id)
reply_post_id = reply_post_id.replace('.json', '') reply_post_id = reply_post_id.replace('.json', '')
reply_post_id = reply_post_id.replace('.replies', '') reply_post_id = reply_post_id.replace('.replies', '')
post_filename = acct_dir(base_dir, nickname, domain) + \ post_filename = acct_dir(base_dir, nickname, domain) + \

3
bookmarks.py
View File

@ -29,6 +29,7 @@ from utils import local_actor_url
from utils import has_actor from utils import has_actor
from utils import has_object_string_type from utils import has_object_string_type
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from posts import get_person_box from posts import get_person_box
from session import post_json from session import post_json
@ -71,7 +72,7 @@ def undo_bookmarks_collection_entry(recent_posts_cache: {},
bookmark_index = post_filename.split('/')[-1].strip() bookmark_index = post_filename.split('/')[-1].strip()
else: else:
bookmark_index = post_filename.strip() bookmark_index = post_filename.strip()
bookmark_index = bookmark_index.replace('\n', '').replace('\r', '') bookmark_index = remove_eol(bookmark_index)
if not text_in_file(bookmark_index, bookmarks_index_filename): if not text_in_file(bookmark_index, bookmarks_index_filename):
return return
index_str = '' index_str = ''

6
city.py
View File

@ -13,6 +13,7 @@ import random
import math import math
from random import randint from random import randint
from utils import acct_dir from utils import acct_dir
from utils import remove_eol
# states which the simulated city dweller can be in # states which the simulated city dweller can be in
PERSON_SLEEP = 0 PERSON_SLEEP = 0
@ -145,7 +146,7 @@ def _get_city_pulse(curr_time_of_day, decoy_seed: int) -> (float, float):
def parse_nogo_string(nogo_line: str) -> []: def parse_nogo_string(nogo_line: str) -> []:
"""Parses a line from locations_nogo.txt and returns the polygon """Parses a line from locations_nogo.txt and returns the polygon
""" """
nogo_line = nogo_line.replace('\n', '').replace('\r', '') nogo_line = remove_eol(nogo_line)
polygon_str = nogo_line.split(':', 1)[1] polygon_str = nogo_line.split(':', 1)[1]
if ';' in polygon_str: if ';' in polygon_str:
pts = polygon_str.split(';') pts = polygon_str.split(';')
@ -320,7 +321,8 @@ def get_spoofed_city(city: str, base_dir: str,
if os.path.isfile(city_filename): if os.path.isfile(city_filename):
try: try:
with open(city_filename, 'r', encoding='utf-8') as city_file: with open(city_filename, 'r', encoding='utf-8') as city_file:
city = city_file.read().replace('\n', '') city1 = city_file.read()
city = remove_eol(city1)
except OSError: except OSError:
print('EX: unable to read ' + city_filename) print('EX: unable to read ' + city_filename)
return city return city

14
content.py
View File

@ -32,6 +32,7 @@ from utils import acct_dir
from utils import is_float from utils import is_float
from utils import get_currencies from utils import get_currencies
from utils import remove_html from utils import remove_html
from utils import remove_eol
from petnames import get_pet_name from petnames import get_pet_name
from session import download_image from session import download_image
@ -288,7 +289,7 @@ def switch_words(base_dir: str, nickname: str, domain: str, content: str,
print('EX: unable to read switches ' + switch_words_filename) print('EX: unable to read switches ' + switch_words_filename)
for line in rules: for line in rules:
replace_str = line.replace('\n', '').replace('\r', '') replace_str = remove_eol(line)
splitters = ('->', ':', ',', ';', '-') splitters = ('->', ':', ',', ';', '-')
word_transform = None word_transform = None
for split_str in splitters: for split_str in splitters:
@ -397,7 +398,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None:
line = count_str + ' ' + emoji_content line = count_str + ' ' + emoji_content
new_common_emoji.append(line) new_common_emoji.append(line)
else: else:
new_common_emoji.append(line.replace('\n', '')) line1 = remove_eol(line)
new_common_emoji.append(line1)
if not emoji_found: if not emoji_found:
new_common_emoji.append(str(1).zfill(16) + ' ' + emoji_content) new_common_emoji.append(str(1).zfill(16) + ' ' + emoji_content)
new_common_emoji.sort(reverse=True) new_common_emoji.sort(reverse=True)
@ -757,7 +759,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
continue continue
follow_nick = follow.split('@')[0] follow_nick = follow.split('@')[0]
if possible_nickname == follow_nick: if possible_nickname == follow_nick:
follow_str = follow.replace('\n', '').replace('\r', '') follow_str = remove_eol(follow)
replace_domain = follow_str.split('@')[1] replace_domain = follow_str.split('@')[1]
recipient_actor = http_prefix + "://" + \ recipient_actor = http_prefix + "://" + \
replace_domain + "/@" + possible_nickname replace_domain + "/@" + possible_nickname
@ -780,10 +782,10 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
if '@' not in follow: if '@' not in follow:
follow_ctr += 1 follow_ctr += 1
continue continue
pet = petnames[follow_ctr].replace('\n', '') pet = remove_eol(petnames[follow_ctr])
if pet: if pet:
if possible_nickname == pet: if possible_nickname == pet:
follow_str = follow.replace('\n', '').replace('\r', '') follow_str = remove_eol(follow)
replace_nickname = follow_str.split('@')[0] replace_nickname = follow_str.split('@')[0]
replace_domain = follow_str.split('@')[1] replace_domain = follow_str.split('@')[1]
recipient_actor = http_prefix + "://" + \ recipient_actor = http_prefix + "://" + \
@ -817,7 +819,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
return False return False
if following: if following:
for follow in following: for follow in following:
if follow.replace('\n', '').replace('\r', '') != possible_handle: if remove_eol(follow) != possible_handle:
continue continue
recipient_actor = http_prefix + "://" + \ recipient_actor = http_prefix + "://" + \
possible_domain + "/@" + possible_nickname possible_domain + "/@" + possible_nickname

3
crawlers.py
View File

@ -11,6 +11,7 @@ import os
import time import time
from utils import save_json from utils import save_json
from utils import user_agent_domain from utils import user_agent_domain
from utils import remove_eol
from blocking import update_blocked_cache from blocking import update_blocked_cache
from blocking import is_blocked_domain from blocking import is_blocked_domain
@ -72,7 +73,7 @@ def load_known_web_bots(base_dir: str) -> []:
for crawler in crawlers_list: for crawler in crawlers_list:
if not crawler: if not crawler:
continue continue
crawler = crawler.replace('\n', '').strip() crawler = remove_eol(crawler).strip()
if not crawler: if not crawler:
continue continue
if crawler not in known_bots: if crawler not in known_bots:

3
daemon.py
View File

@ -253,6 +253,7 @@ from languages import set_actor_languages
from languages import get_understood_languages from languages import get_understood_languages
from like import update_likes_collection from like import update_likes_collection
from reaction import update_reaction_collection from reaction import update_reaction_collection
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import is_onion_request from utils import is_onion_request
from utils import is_i2p_request from utils import is_i2p_request
@ -4185,7 +4186,7 @@ class PubServer(BaseHTTPRequestHandler):
timezone = \ timezone = \
self.server.account_timezone.get(nickname) self.server.account_timezone.get(nickname)
profile_handle = search_str.replace('\n', '').strip() profile_handle = remove_eol(search_str).strip()
# establish the session # establish the session
curr_proxy_type = proxy_type curr_proxy_type = proxy_type

84
epicyon.py
View File

@ -68,6 +68,7 @@ from tests import test_update_actor
from tests import run_all_tests from tests import run_all_tests
from auth import store_basic_credentials from auth import store_basic_credentials
from auth import create_password from auth import create_password
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import remove_domain_port from utils import remove_domain_port
from utils import get_port_from_domain from utils import get_port_from_domain
@ -1457,7 +1458,8 @@ def _command_options() -> None:
with open(approve_follows_filename, 'r', with open(approve_follows_filename, 'r',
encoding='utf-8') as approvefile: encoding='utf-8') as approvefile:
for approve in approvefile: for approve in approvefile:
print(approve.replace('\n', '').replace('\r', '')) approve1 = remove_eol(approve)
print(approve1)
approve_ctr += 1 approve_ctr += 1
if approve_ctr == 0: if approve_ctr == 0:
print('There are no follow requests pending approval.') print('There are no follow requests pending approval.')
@ -1488,7 +1490,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
if not argb.sendto: if not argb.sendto:
@ -1504,7 +1506,7 @@ def _command_options() -> None:
if '@' in argb.sendto: if '@' in argb.sendto:
to_nickname = argb.sendto.split('@')[0] to_nickname = argb.sendto.split('@')[0]
to_domain = argb.sendto.split('@')[1] to_domain = argb.sendto.split('@')[1]
to_domain = to_domain.replace('\n', '').replace('\r', '') to_domain = remove_eol(to_domain)
to_port = 443 to_port = 443
if ':' in to_domain: if ':' in to_domain:
to_port = get_port_from_domain(to_domain) to_port = get_port_from_domain(to_domain)
@ -1583,7 +1585,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
proxy_type = None proxy_type = None
if argb.tor or domain.endswith('.onion'): if argb.tor or domain.endswith('.onion'):
proxy_type = 'tor' proxy_type = 'tor'
@ -1624,7 +1626,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -1661,7 +1663,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
proxy_type = None proxy_type = None
if argb.tor or domain.endswith('.onion'): if argb.tor or domain.endswith('.onion'):
@ -1697,7 +1699,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if not argb.nickname: if not argb.nickname:
print('Specify a nickname with the --nickname option') print('Specify a nickname with the --nickname option')
@ -1769,7 +1771,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if not argb.nickname: if not argb.nickname:
print('Specify a nickname with the --nickname option') print('Specify a nickname with the --nickname option')
@ -1803,7 +1805,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if not argb.nickname: if not argb.nickname:
print('Specify a nickname with the --nickname option') print('Specify a nickname with the --nickname option')
@ -1875,7 +1877,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if not argb.nickname: if not argb.nickname:
print('Specify a nickname with the --nickname option') print('Specify a nickname with the --nickname option')
@ -1913,7 +1915,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -1952,7 +1954,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -1985,7 +1987,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2025,7 +2027,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2060,7 +2062,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2094,7 +2096,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2127,7 +2129,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2164,7 +2166,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
follow_nickname = get_nickname_from_actor(argb.follow) follow_nickname = get_nickname_from_actor(argb.follow)
if not follow_nickname: if not follow_nickname:
@ -2212,7 +2214,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
follow_nickname = get_nickname_from_actor(argb.unfollow) follow_nickname = get_nickname_from_actor(argb.unfollow)
if not follow_nickname: if not follow_nickname:
@ -2257,7 +2259,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2290,7 +2292,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2324,7 +2326,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2442,31 +2444,31 @@ def _command_options() -> None:
sys.exit() sys.exit()
if '/users/' in argb.followers: if '/users/' in argb.followers:
nickname = argb.followers.split('/users/')[1] nickname = argb.followers.split('/users/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/users/')[0] domain = argb.followers.split('/users/')[0]
elif '/profile/' in argb.followers: elif '/profile/' in argb.followers:
nickname = argb.followers.split('/profile/')[1] nickname = argb.followers.split('/profile/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/profile/')[0] domain = argb.followers.split('/profile/')[0]
elif '/author/' in argb.followers: elif '/author/' in argb.followers:
nickname = argb.followers.split('/author/')[1] nickname = argb.followers.split('/author/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/author/')[0] domain = argb.followers.split('/author/')[0]
elif '/channel/' in argb.followers: elif '/channel/' in argb.followers:
nickname = argb.followers.split('/channel/')[1] nickname = argb.followers.split('/channel/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/channel/')[0] domain = argb.followers.split('/channel/')[0]
elif '/accounts/' in argb.followers: elif '/accounts/' in argb.followers:
nickname = argb.followers.split('/accounts/')[1] nickname = argb.followers.split('/accounts/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/accounts/')[0] domain = argb.followers.split('/accounts/')[0]
elif '/u/' in argb.followers: elif '/u/' in argb.followers:
nickname = argb.followers.split('/u/')[1] nickname = argb.followers.split('/u/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/u/')[0] domain = argb.followers.split('/u/')[0]
elif '/c/' in argb.followers: elif '/c/' in argb.followers:
nickname = argb.followers.split('/c/')[1] nickname = argb.followers.split('/c/')[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = argb.followers.split('/c/')[0] domain = argb.followers.split('/c/')[0]
else: else:
# format: @nick@domain # format: @nick@domain
@ -2480,7 +2482,7 @@ def _command_options() -> None:
sys.exit() sys.exit()
nickname = argb.followers.split('@')[0] nickname = argb.followers.split('@')[0]
domain = argb.followers.split('@')[1] domain = argb.followers.split('@')[1]
domain = domain.replace('\n', '').replace('\r', '') domain = remove_eol(domain)
cached_webfingers = {} cached_webfingers = {}
if argb.http or domain.endswith('.onion'): if argb.http or domain.endswith('.onion'):
http_prefix = 'http' http_prefix = 'http'
@ -2603,7 +2605,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if len(argb.password.strip()) < 8: if len(argb.password.strip()) < 8:
print('Password should be at least 8 characters') print('Password should be at least 8 characters')
sys.exit() sys.exit()
@ -2652,7 +2654,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if len(argb.password.strip()) < 8: if len(argb.password.strip()) < 8:
print('Password should be at least 8 characters') print('Password should be at least 8 characters')
sys.exit() sys.exit()
@ -2818,7 +2820,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if not argb.skillLevelPercent: if not argb.skillLevelPercent:
print('Specify a skill level in the range 0-100') print('Specify a skill level in the range 0-100')
@ -2862,7 +2864,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -2906,7 +2908,7 @@ def _command_options() -> None:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
proxy_type = None proxy_type = None
if argb.tor or domain.endswith('.onion'): if argb.tor or domain.endswith('.onion'):
@ -2965,11 +2967,11 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if '@' in argb.block: if '@' in argb.block:
blocked_domain = argb.block.split('@')[1] blocked_domain = argb.block.split('@')[1]
blocked_domain = blocked_domain.replace('\n', '').replace('\r', '') blocked_domain = remove_eol(blocked_domain)
blocked_nickname = argb.block.split('@')[0] blocked_nickname = argb.block.split('@')[0]
blocked_actor = http_prefix + '://' + blocked_domain + \ blocked_actor = http_prefix + '://' + blocked_domain + \
'/users/' + blocked_nickname '/users/' + blocked_nickname
@ -3009,7 +3011,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -3041,7 +3043,7 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
session = create_session(proxy_type) session = create_session(proxy_type)
person_cache = {} person_cache = {}
@ -3073,11 +3075,11 @@ def _command_options() -> None:
if not argb.password: if not argb.password:
print('Specify a password with the --password option') print('Specify a password with the --password option')
sys.exit() sys.exit()
argb.password = argb.password.replace('\n', '') argb.password = remove_eol(argb.password)
if '@' in argb.unblock: if '@' in argb.unblock:
blocked_domain = argb.unblock.split('@')[1] blocked_domain = argb.unblock.split('@')[1]
blocked_domain = blocked_domain.replace('\n', '').replace('\r', '') blocked_domain = remove_eol(blocked_domain)
blocked_nickname = argb.unblock.split('@')[0] blocked_nickname = argb.unblock.split('@')[0]
blocked_actor = http_prefix + '://' + blocked_domain + \ blocked_actor = http_prefix + '://' + blocked_domain + \
'/users/' + blocked_nickname '/users/' + blocked_nickname

7
filters.py
View File

@ -10,6 +10,7 @@ __module_group__ = "Moderation"
import os import os
from utils import acct_dir from utils import acct_dir
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool: def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool:
@ -62,7 +63,7 @@ def remove_filter(base_dir: str, nickname: str, domain: str,
with open(filters_filename, 'r', encoding='utf-8') as fp_filt: with open(filters_filename, 'r', encoding='utf-8') as fp_filt:
with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew:
for line in fp_filt: for line in fp_filt:
line = line.replace('\n', '') line = remove_eol(line)
if line != words: if line != words:
fpnew.write(line + '\n') fpnew.write(line + '\n')
except OSError as ex: except OSError as ex:
@ -87,7 +88,7 @@ def remove_global_filter(base_dir: str, words: str) -> bool:
with open(filters_filename, 'r', encoding='utf-8') as fp_filt: with open(filters_filename, 'r', encoding='utf-8') as fp_filt:
with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew: with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew:
for line in fp_filt: for line in fp_filt:
line = line.replace('\n', '') line = remove_eol(line)
if line != words: if line != words:
fpnew.write(line + '\n') fpnew.write(line + '\n')
except OSError as ex: except OSError as ex:
@ -122,7 +123,7 @@ def _is_filtered_base(filename: str, content: str) -> bool:
try: try:
with open(filename, 'r', encoding='utf-8') as fp_filt: with open(filename, 'r', encoding='utf-8') as fp_filt:
for line in fp_filt: for line in fp_filt:
filter_str = line.replace('\n', '').replace('\r', '') filter_str = remove_eol(line)
if not filter_str: if not filter_str:
continue continue
if len(filter_str) < 2: if len(filter_str) < 2:

13
follow.py
View File

@ -31,6 +31,7 @@ from utils import acct_dir
from utils import has_group_type from utils import has_group_type
from utils import local_actor_url from utils import local_actor_url
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from acceptreject import create_accept from acceptreject import create_accept
from acceptreject import create_reject from acceptreject import create_reject
from webfinger import webfinger_handle from webfinger import webfinger_handle
@ -67,7 +68,7 @@ def create_initial_last_seen(base_dir: str, http_prefix: str) -> None:
continue continue
if '@' not in handle: if '@' not in handle:
continue continue
handle = handle.replace('\n', '') handle = remove_eol(handle)
nickname = handle.split('@')[0] nickname = handle.split('@')[0]
domain = handle.split('@')[1] domain = handle.split('@')[1]
if nickname.startswith('!'): if nickname.startswith('!'):
@ -244,7 +245,7 @@ def get_follower_domains(base_dir: str, nickname: str, domain: str) -> []:
domains_list = [] domains_list = []
for handle in lines: for handle in lines:
handle = handle.replace('\n', '') handle = remove_eol(handle)
follower_domain, _ = get_domain_from_actor(handle) follower_domain, _ = get_domain_from_actor(handle)
if not follower_domain: if not follower_domain:
continue continue
@ -535,8 +536,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str,
page_ctr += 1 page_ctr += 1
total_ctr += 1 total_ctr += 1
if curr_page == page_number: if curr_page == page_number:
line2 = \ line2_lower = line.lower()
line.lower().replace('\n', '').replace('\r', '') line2 = remove_eol(line2_lower)
nick = line2.split('@')[0] nick = line2.split('@')[0]
dom = line2.split('@')[1] dom = line2.split('@')[1]
if not nick.startswith('!'): if not nick.startswith('!'):
@ -555,8 +556,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str,
page_ctr += 1 page_ctr += 1
total_ctr += 1 total_ctr += 1
if curr_page == page_number: if curr_page == page_number:
append_str = \ append_str1 = line.lower()
line.lower().replace('\n', '').replace('\r', '') append_str = remove_eol(append_str1)
following['orderedItems'].append(append_str) following['orderedItems'].append(append_str)
if page_ctr >= follows_per_page: if page_ctr >= follows_per_page:
page_ctr = 0 page_ctr = 0

9
happening.py
View File

@ -25,6 +25,7 @@ from utils import delete_post
from utils import get_status_number from utils import get_status_number
from utils import get_full_domain from utils import get_full_domain
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from filters import is_filtered from filters import is_filtered
from context import get_individual_post_context from context import get_individual_post_context
from session import get_method from session import get_method
@ -262,7 +263,7 @@ def get_todays_events(base_dir: str, nickname: str, domain: str,
recreate_events_file = False recreate_events_file = False
with open(calendar_filename, 'r', encoding='utf-8') as events_file: with open(calendar_filename, 'r', encoding='utf-8') as events_file:
for post_id in events_file: for post_id in events_file:
post_id = post_id.replace('\n', '').replace('\r', '') post_id = remove_eol(post_id)
post_filename = locate_post(base_dir, nickname, domain, post_id) post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename: if not post_filename:
recreate_events_file = True recreate_events_file = True
@ -556,7 +557,7 @@ def day_events_check(base_dir: str, nickname: str, domain: str,
events_exist = False events_exist = False
with open(calendar_filename, 'r', encoding='utf-8') as events_file: with open(calendar_filename, 'r', encoding='utf-8') as events_file:
for post_id in events_file: for post_id in events_file:
post_id = post_id.replace('\n', '').replace('\r', '') post_id = remove_eol(post_id)
post_filename = locate_post(base_dir, nickname, domain, post_id) post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename: if not post_filename:
continue continue
@ -612,7 +613,7 @@ def get_this_weeks_events(base_dir: str, nickname: str, domain: str) -> {}:
recreate_events_file = False recreate_events_file = False
with open(calendar_filename, 'r', encoding='utf-8') as events_file: with open(calendar_filename, 'r', encoding='utf-8') as events_file:
for post_id in events_file: for post_id in events_file:
post_id = post_id.replace('\n', '').replace('\r', '') post_id = remove_eol(post_id)
post_filename = locate_post(base_dir, nickname, domain, post_id) post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename: if not post_filename:
recreate_events_file = True recreate_events_file = True
@ -679,7 +680,7 @@ def get_calendar_events(base_dir: str, nickname: str, domain: str,
recreate_events_file = False recreate_events_file = False
with open(calendar_filename, 'r', encoding='utf-8') as events_file: with open(calendar_filename, 'r', encoding='utf-8') as events_file:
for post_id in events_file: for post_id in events_file:
post_id = post_id.replace('\n', '').replace('\r', '') post_id = remove_eol(post_id)
post_filename = locate_post(base_dir, nickname, domain, post_id) post_filename = locate_post(base_dir, nickname, domain, post_id)
if not post_filename: if not post_filename:
recreate_events_file = True recreate_events_file = True

4
inbox.py
View File

@ -18,6 +18,7 @@ from languages import understood_post_language
from like import update_likes_collection from like import update_likes_collection
from reaction import update_reaction_collection from reaction import update_reaction_collection
from reaction import valid_emoji_content from reaction import valid_emoji_content
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import get_media_descriptions_from_post from utils import get_media_descriptions_from_post
from utils import get_summary_from_post from utils import get_summary_from_post
@ -2314,7 +2315,8 @@ def _receive_announce(recent_posts_cache: {},
str(message_json)) str(message_json))
else: else:
if debug: if debug:
print('Generated announce html ' + announce_html.replace('\n', '')) announce_html2 = remove_eol(announce_html)
print('Generated announce html ' + announce_html2)
post_json_object = download_announce(session, base_dir, post_json_object = download_announce(session, base_dir,
http_prefix, http_prefix,

8
manualapprove.py
View File

@ -17,6 +17,7 @@ from utils import get_port_from_domain
from utils import get_user_paths from utils import get_user_paths
from utils import acct_dir from utils import acct_dir
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from threads import thread_with_trace from threads import thread_with_trace
from session import create_session from session import create_session
@ -57,8 +58,7 @@ def manual_deny_follow_request(session, session_onion, session_i2p,
print('EX: unable to append ' + rejected_follows_filename) print('EX: unable to append ' + rejected_follows_filename)
deny_nickname = deny_handle.split('@')[0] deny_nickname = deny_handle.split('@')[0]
deny_domain = \ deny_domain = remove_eol(deny_handle.split('@')[1])
deny_handle.split('@')[1].replace('\n', '').replace('\r', '')
deny_port = port deny_port = port
if ':' in deny_domain: if ':' in deny_domain:
deny_port = get_port_from_domain(deny_domain) deny_port = get_port_from_domain(deny_domain)
@ -195,7 +195,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
# is this the approved follow? # is this the approved follow?
if handle_of_follow_requester.startswith(approve_handle_full): if handle_of_follow_requester.startswith(approve_handle_full):
handle_of_follow_requester = \ handle_of_follow_requester = \
handle_of_follow_requester.replace('\n', '') remove_eol(handle_of_follow_requester)
handle_of_follow_requester = \ handle_of_follow_requester = \
handle_of_follow_requester.replace('\r', '') handle_of_follow_requester.replace('\r', '')
port2 = port port2 = port
@ -212,7 +212,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
approve_nickname = approve_handle.split('@')[0] approve_nickname = approve_handle.split('@')[0]
approve_domain = approve_handle.split('@')[1] approve_domain = approve_handle.split('@')[1]
approve_domain = \ approve_domain = \
approve_domain.replace('\n', '') remove_eol(approve_domain)
approve_domain = \ approve_domain = \
approve_domain.replace('\r', '') approve_domain.replace('\r', '')
approve_port = port2 approve_port = port2

4
newswire.py
View File

@ -20,6 +20,7 @@ from datetime import timezone
from collections import OrderedDict from collections import OrderedDict
from utils import valid_post_date from utils import valid_post_date
from categories import set_hashtag_category from categories import set_hashtag_category
from utils import remove_eol
from utils import get_domain_from_actor from utils import get_domain_from_actor
from utils import valid_hash_tag from utils import valid_hash_tag
from utils import dangerous_svg from utils import dangerous_svg
@ -1458,8 +1459,7 @@ def _add_account_blogs_to_newswire(base_dir: str, nickname: str, domain: str,
# filename of the post without any extension or path # filename of the post without any extension or path
# This should also correspond to any index entry in # This should also correspond to any index entry in
# the posts cache # the posts cache
post_url = \ post_url = remove_eol(post_filename)
post_filename.replace('\n', '').replace('\r', '')
post_url = post_url.replace('.json', '').strip() post_url = post_url.replace('.json', '').strip()
# read the post from file # read the post from file

13
person.py
View File

@ -43,7 +43,7 @@ from utils import get_nickname_from_actor
from utils import remove_html from utils import remove_html
from utils import contains_invalid_chars from utils import contains_invalid_chars
from utils import replace_users_with_at from utils import replace_users_with_at
from utils import remove_line_endings from utils import remove_eol
from utils import remove_domain_port from utils import remove_domain_port
from utils import get_status_number from utils import get_status_number
from utils import get_full_domain from utils import get_full_domain
@ -63,6 +63,7 @@ from utils import get_group_paths
from utils import local_actor_url from utils import local_actor_url
from utils import dangerous_svg from utils import dangerous_svg
from utils import text_in_file from utils import text_in_file
from utils import remove_line_endings
from session import create_session from session import create_session
from session import get_json from session import get_json
from webfinger import webfinger_handle from webfinger import webfinger_handle
@ -104,7 +105,7 @@ def set_profile_image(base_dir: str, http_prefix: str,
"""Saves the given image file as an avatar or background """Saves the given image file as an avatar or background
image for the given person image for the given person
""" """
image_filename = image_filename.replace('\n', '').replace('\r', '') image_filename = remove_eol(image_filename)
if not is_image_file(image_filename): if not is_image_file(image_filename):
print('Profile image must be png, jpg, gif or svg format') print('Profile image must be png, jpg, gif or svg format')
return False return False
@ -1371,8 +1372,8 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str,
for line in snoozed_file: for line in snoozed_file:
# is this the entry for the actor? # is this the entry for the actor?
if line.startswith(snooze_actor + ' '): if line.startswith(snooze_actor + ' '):
snoozed_time_str = \ snoozed_time_str1 = line.split(' ')[1]
line.split(' ')[1].replace('\n', '').replace('\r', '') snoozed_time_str = remove_eol(snoozed_time_str1)
# is there a time appended? # is there a time appended?
if snoozed_time_str.isdigit(): if snoozed_time_str.isdigit():
snoozed_time = int(snoozed_time_str) snoozed_time = int(snoozed_time_str)
@ -1525,7 +1526,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool,
for user_path in paths: for user_path in paths:
if user_path in handle: if user_path in handle:
nickname = handle.split(user_path)[1] nickname = handle.split(user_path)[1]
nickname = nickname.replace('\n', '').replace('\r', '') nickname = remove_eol(nickname)
domain = handle.split(user_path)[0] domain = handle.split(user_path)[0]
user_path_found = True user_path_found = True
break break
@ -1556,7 +1557,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool,
return None, None return None, None
nickname = handle.split('@')[0] nickname = handle.split('@')[0]
domain = handle.split('@')[1] domain = handle.split('@')[1]
domain = domain.replace('\n', '').replace('\r', '') domain = remove_eol(domain)
cached_webfingers = {} cached_webfingers = {}
proxy_type = None proxy_type = None

15
posts.py
View File

@ -32,6 +32,7 @@ from webfinger import webfinger_handle
from httpsig import create_signed_header from httpsig import create_signed_header
from siteactive import site_is_active from siteactive import site_is_active
from languages import understood_post_language from languages import understood_post_language
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import get_media_descriptions_from_post from utils import get_media_descriptions_from_post
from utils import valid_hash_tag from utils import valid_hash_tag
@ -139,8 +140,7 @@ def no_of_followers_on_domain(base_dir: str, handle: str,
for follower_handle in followers_file: for follower_handle in followers_file:
if '@' in follower_handle: if '@' in follower_handle:
follower_domain = follower_handle.split('@')[1] follower_domain = follower_handle.split('@')[1]
follower_domain = follower_domain.replace('\n', '') follower_domain = remove_eol(follower_domain)
follower_domain = follower_domain.replace('\r', '')
if domain == follower_domain: if domain == follower_domain:
ctr += 1 ctr += 1
return ctr return ctr
@ -2707,8 +2707,8 @@ def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}:
for follower_handle in foll_file: for follower_handle in foll_file:
if '@' not in follower_handle: if '@' not in follower_handle:
continue continue
fhandle = \ fhandle1 = follower_handle.strip()
follower_handle.strip().replace('\n', '').replace('\r', '') fhandle = remove_eol(fhandle1)
follower_domain = fhandle.split('@')[1] follower_domain = fhandle.split('@')[1]
if not grouped.get(follower_domain): if not grouped.get(follower_domain):
grouped[follower_domain] = [fhandle] grouped[follower_domain] = [fhandle]
@ -4041,8 +4041,7 @@ def _create_box_indexed(recent_posts_cache: {},
# filename of the post without any extension or path # filename of the post without any extension or path
# This should also correspond to any index entry in # This should also correspond to any index entry in
# the posts cache # the posts cache
post_url = \ post_url = remove_eol(post_filename)
post_filename.replace('\n', '').replace('\r', '')
post_url = post_url.replace('.json', '').strip() post_url = post_url.replace('.json', '').strip()
if post_url in post_urls_in_box: if post_url in post_urls_in_box:
@ -4737,7 +4736,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str,
reply_found = False reply_found = False
# examine inbox and outbox # examine inbox and outbox
for boxname in replies_boxes: for boxname in replies_boxes:
message_id2 = message_id.replace('\n', '').replace('\r', '') message_id2 = remove_eol(message_id)
search_filename = \ search_filename = \
acct_dir(base_dir, nickname, domain) + '/' + \ acct_dir(base_dir, nickname, domain) + '/' + \
boxname + '/' + \ boxname + '/' + \
@ -4763,7 +4762,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str,
break break
# if not in either inbox or outbox then examine the shared inbox # if not in either inbox or outbox then examine the shared inbox
if not reply_found: if not reply_found:
message_id2 = message_id.replace('\n', '').replace('\r', '') message_id2 = remove_eol(message_id)
search_filename = \ search_filename = \
base_dir + \ base_dir + \
'/accounts/inbox@' + \ '/accounts/inbox@' + \

4
reaction.py
View File

@ -31,6 +31,7 @@ from utils import save_json
from utils import remove_post_from_cache from utils import remove_post_from_cache
from utils import get_cached_post_filename from utils import get_cached_post_filename
from utils import contains_invalid_chars from utils import contains_invalid_chars
from utils import remove_eol
from posts import send_signed_json from posts import send_signed_json
from session import post_json from session import post_json
from webfinger import webfinger_handle from webfinger import webfinger_handle
@ -470,7 +471,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None:
line = count_str + ' ' + emoji_content line = count_str + ' ' + emoji_content
new_common_reactions.append(line) new_common_reactions.append(line)
else: else:
new_common_reactions.append(line.replace('\n', '')) line1 = remove_eol(line)
new_common_reactions.append(line1)
if not reaction_found: if not reaction_found:
new_common_reactions.append(str(1).zfill(16) + ' ' + emoji_content) new_common_reactions.append(str(1).zfill(16) + ' ' + emoji_content)
new_common_reactions.sort(reverse=True) new_common_reactions.sort(reverse=True)

4
schedule.py
View File

@ -15,6 +15,7 @@ from utils import get_status_number
from utils import load_json from utils import load_json
from utils import is_account_dir from utils import is_account_dir
from utils import acct_dir from utils import acct_dir
from utils import remove_eol
from outbox import post_message_to_outbox from outbox import post_message_to_outbox
from session import create_session from session import create_session
@ -43,7 +44,8 @@ def _update_post_schedule(base_dir: str, handle: str, httpd,
date_str = line.split(' ')[0] date_str = line.split(' ')[0]
if 'T' not in date_str: if 'T' not in date_str:
continue continue
post_id = line.split(' ', 1)[1].replace('\n', '').replace('\r', '') post_id1 = line.split(' ', 1)[1]
post_id = remove_eol(post_id1)
post_filename = schedule_dir + post_id + '.json' post_filename = schedule_dir + post_id + '.json'
if delete_schedule_post: if delete_schedule_post:
# delete extraneous scheduled posts # delete extraneous scheduled posts

3
shares.py
View File

@ -22,6 +22,7 @@ from posts import get_person_box
from session import post_json from session import post_json
from session import post_image from session import post_image
from session import create_session from session import create_session
from utils import remove_eol
from utils import has_object_string_type from utils import has_object_string_type
from utils import date_string_to_seconds from utils import date_string_to_seconds
from utils import date_seconds_to_string from utils import date_seconds_to_string
@ -1502,7 +1503,7 @@ def authorize_shared_items(shared_items_federated_domains: [],
if debug: if debug:
print('DEBUG: shared item federation should not use basic auth') print('DEBUG: shared item federation should not use basic auth')
return False return False
provided_token = auth_header.replace('\n', '').replace('\r', '').strip() provided_token = remove_eol(auth_header).strip()
if not provided_token: if not provided_token:
if debug: if debug:
print('DEBUG: shared item federation token is empty') print('DEBUG: shared item federation token is empty')

19
tests.py
View File

@ -54,6 +54,7 @@ from follow import clear_followers
from follow import send_follow_request_via_server from follow import send_follow_request_via_server
from follow import send_unfollow_request_via_server from follow import send_unfollow_request_via_server
from siteactive import site_is_active from siteactive import site_is_active
from utils import remove_eol
from utils import text_in_file from utils import text_in_file
from utils import convert_published_to_local_timezone from utils import convert_published_to_local_timezone
from utils import convert_to_snake_case from utils import convert_to_snake_case
@ -2741,7 +2742,7 @@ def _test_follows(base_dir: str) -> None:
domain_found = False domain_found = False
for following_domain in fp_foll: for following_domain in fp_foll:
test_domain = following_domain.split('@')[1] test_domain = following_domain.split('@')[1]
test_domain = test_domain.replace('\n', '').replace('\r', '') test_domain = remove_eol(test_domain)
if test_domain == 'mesh.com': if test_domain == 'mesh.com':
domain_found = True domain_found = True
if test_domain not in federation_list: if test_domain not in federation_list:
@ -2755,7 +2756,7 @@ def _test_follows(base_dir: str) -> None:
domain_found = False domain_found = False
for following_domain in fp_foll: for following_domain in fp_foll:
test_domain = following_domain.split('@')[1] test_domain = following_domain.split('@')[1]
test_domain = test_domain.replace('\n', '').replace('\r', '') test_domain = remove_eol(test_domain)
if test_domain == 'mesh.com': if test_domain == 'mesh.com':
domain_found = True domain_found = True
assert domain_found is False assert domain_found is False
@ -2779,7 +2780,7 @@ def _test_follows(base_dir: str) -> None:
encoding='utf-8') as fp_foll: encoding='utf-8') as fp_foll:
for follower_domain in fp_foll: for follower_domain in fp_foll:
test_domain = follower_domain.split('@')[1] test_domain = follower_domain.split('@')[1]
test_domain = test_domain.replace('\n', '').replace('\r', '') test_domain = remove_eol(test_domain)
if test_domain not in federation_list: if test_domain not in federation_list:
print(test_domain) print(test_domain)
assert False assert False
@ -4715,7 +4716,7 @@ def _get_function_call_args(name: str, lines: [], start_line_ctr: int) -> []:
continue continue
args_str += lines[line_ctr].split(')')[0] args_str += lines[line_ctr].split(')')[0]
break break
return args_str.replace('\n', '').replace(' ', '').split(',') return remove_eol(args_str).replace(' ', '').split(',')
def get_function_calls(name: str, lines: [], start_line_ctr: int, def get_function_calls(name: str, lines: [], start_line_ctr: int,
@ -7257,6 +7258,15 @@ def _test_color_contrast_value(base_dir: str) -> None:
print('Color contrast is ok for all themes') print('Color contrast is ok for all themes')
def _test_remove_end_of_line():
print('remove_end_of_line')
text = 'some text\r\n'
expected = 'some text'
assert remove_eol(text) == expected
text = 'some text'
assert remove_eol(text) == expected
def run_all_tests(): def run_all_tests():
base_dir = os.getcwd() base_dir = os.getcwd()
print('Running tests...') print('Running tests...')
@ -7274,6 +7284,7 @@ def run_all_tests():
_test_checkbox_names() _test_checkbox_names()
_test_thread_functions() _test_thread_functions()
_test_functions() _test_functions()
_test_remove_end_of_line()
_test_translation_labels() _test_translation_labels()
_test_color_contrast_value(base_dir) _test_color_contrast_value(base_dir)
_test_diff_content() _test_diff_content()

4
theme.py
View File

@ -18,6 +18,7 @@ from utils import dangerous_svg
from utils import local_actor_url from utils import local_actor_url
from utils import remove_html from utils import remove_html
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from shutil import copyfile from shutil import copyfile
from shutil import make_archive from shutil import make_archive
from shutil import unpack_archive from shutil import unpack_archive
@ -44,7 +45,8 @@ def import_theme(base_dir: str, filename: str) -> bool:
new_theme_name = None new_theme_name = None
with open(temp_theme_dir + '/name.txt', 'r', with open(temp_theme_dir + '/name.txt', 'r',
encoding='utf-8') as fp_theme: encoding='utf-8') as fp_theme:
new_theme_name = fp_theme.read().replace('\n', '').replace('\r', '') new_theme_name1 = fp_theme.read()
new_theme_name = remove_eol(new_theme_name1)
if len(new_theme_name) > 20: if len(new_theme_name) > 20:
print('WARN: Imported theme name is too long') print('WARN: Imported theme name is too long')
return False return False

23
utils.py
View File

@ -40,6 +40,12 @@ INVALID_CHARACTERS = (
) )
def remove_eol(line: str):
"""Removes line ending characters
"""
return line.replace('\n', '').replace('\r', '')
def text_in_file(text: str, filename: str, def text_in_file(text: str, filename: str,
case_sensitive: bool = True) -> bool: case_sensitive: bool = True) -> bool:
"""is the given text in the given file? """is the given text in the given file?
@ -671,8 +677,7 @@ def get_followers_of_person(base_dir: str,
continue continue
with open(filename, 'r', encoding='utf-8') as followingfile: with open(filename, 'r', encoding='utf-8') as followingfile:
for following_handle in followingfile: for following_handle in followingfile:
following_handle2 = following_handle.replace('\n', '') following_handle2 = remove_eol(following_handle)
following_handle2 = following_handle2.replace('\r', '')
if following_handle2 == handle: if following_handle2 == handle:
if account not in followers: if account not in followers:
followers.append(account) followers.append(account)
@ -1306,7 +1311,8 @@ def follow_person(base_dir: str, nickname: str, domain: str,
follow_file: str = 'following.txt') -> bool: follow_file: str = 'following.txt') -> bool:
"""Adds a person to the follow list """Adds a person to the follow list
""" """
follow_domain_str_lower = follow_domain.lower().replace('\n', '') follow_domain_str_lower1 = follow_domain.lower()
follow_domain_str_lower = remove_eol(follow_domain_str_lower1)
if not domain_permitted(follow_domain_str_lower, if not domain_permitted(follow_domain_str_lower,
federation_list): federation_list):
if debug: if debug:
@ -1414,8 +1420,8 @@ def locate_news_votes(base_dir: str, domain: str,
"""Returns the votes filename for a news post """Returns the votes filename for a news post
within the news user account within the news user account
""" """
post_url = \ post_url1 = post_url.strip()
post_url.strip().replace('\n', '').replace('\r', '') post_url = remove_eol(post_url1)
# if this post in the shared inbox? # if this post in the shared inbox?
post_url = remove_id_ending(post_url.strip()).replace('/', '#') post_url = remove_id_ending(post_url.strip()).replace('/', '#')
@ -1438,8 +1444,8 @@ def locate_news_arrival(base_dir: str, domain: str,
"""Returns the arrival time for a news post """Returns the arrival time for a news post
within the news user account within the news user account
""" """
post_url = \ post_url1 = post_url.strip()
post_url.strip().replace('\n', '').replace('\r', '') post_url = remove_eol(post_url1)
# if this post in the shared inbox? # if this post in the shared inbox?
post_url = remove_id_ending(post_url.strip()).replace('/', '#') post_url = remove_id_ending(post_url.strip()).replace('/', '#')
@ -2910,8 +2916,7 @@ def reject_post_id(base_dir: str, nickname: str, domain: str,
# filename of the post without any extension or path # filename of the post without any extension or path
# This should also correspond to any index entry in # This should also correspond to any index entry in
# the posts cache # the posts cache
post_url = \ post_url = remove_eol(index_filename)
index_filename.replace('\n', '').replace('\r', '')
post_url = post_url.replace('.json', '').strip() post_url = post_url.replace('.json', '').strip()
if post_url in recent_posts_cache['index']: if post_url in recent_posts_cache['index']:

3
webapp_moderation.py
View File

@ -17,6 +17,7 @@ from utils import get_nickname_from_actor
from utils import get_domain_from_actor from utils import get_domain_from_actor
from utils import get_config_param from utils import get_config_param
from utils import local_actor_url from utils import local_actor_url
from utils import remove_eol
from posts import download_follow_collection from posts import download_follow_collection
from posts import get_public_post_info from posts import get_public_post_info
from posts import is_moderator from posts import is_moderator
@ -418,7 +419,7 @@ def html_moderation_info(translate: {}, base_dir: str,
for line in blocked_lines: for line in blocked_lines:
if not line: if not line:
continue continue
line = line.replace('\n', '').replace('\r', '').strip() line = remove_eol(line).strip()
blocked_str += line + '\n' blocked_str += line + '\n'
info_form += '<div class="container">\n' info_form += '<div class="container">\n'
info_form += \ info_form += \

4
webapp_post.py
View File

@ -25,6 +25,7 @@ from posts import post_is_muted
from posts import get_person_box from posts import get_person_box
from posts import download_announce from posts import download_announce
from posts import populate_replies_json from posts import populate_replies_json
from utils import remove_eol
from utils import disallow_announce from utils import disallow_announce
from utils import disallow_reply from utils import disallow_reply
from utils import convert_published_to_local_timezone from utils import convert_published_to_local_timezone
@ -1542,7 +1543,8 @@ def _substitute_onion_domains(base_dir: str, content: str) -> str:
if sep not in line: if sep not in line:
continue continue
clearnet = line.split(sep, 1)[0].strip() clearnet = line.split(sep, 1)[0].strip()
onion = line.split(sep, 1)[1].strip().replace('\n', '') onion1 = line.split(sep, 1)[1].strip()
onion = remove_eol(onion1)
if clearnet and onion: if clearnet and onion:
onion_domains[clearnet] = onion onion_domains[clearnet] = onion
break break

7
webapp_profile.py
View File

@ -30,6 +30,7 @@ from utils import get_supported_languages
from utils import local_actor_url from utils import local_actor_url
from utils import get_reply_interval_hours from utils import get_reply_interval_hours
from utils import get_account_timezone from utils import get_account_timezone
from utils import remove_eol
from languages import get_actor_languages from languages import get_actor_languages
from skills import get_skills from skills import get_skills
from theme import get_themes_list from theme import get_themes_list
@ -796,7 +797,8 @@ def html_profile(signing_priv_key_pem: str,
encoding='utf-8') as req_file: encoding='utf-8') as req_file:
for follower_handle in req_file: for follower_handle in req_file:
if len(follower_handle) > 0: if len(follower_handle) > 0:
follower_handle = follower_handle.replace('\n', '') follower_handle = \
remove_eol(follower_handle)
if '://' in follower_handle: if '://' in follower_handle:
follower_actor = follower_handle follower_actor = follower_handle
else: else:
@ -1751,7 +1753,8 @@ def _html_edit_profile_filtering(base_dir: str, nickname: str, domain: str,
city_filename = acct_dir(base_dir, nickname, domain) + '/city.txt' city_filename = acct_dir(base_dir, nickname, domain) + '/city.txt'
if os.path.isfile(city_filename): if os.path.isfile(city_filename):
with open(city_filename, 'r', encoding='utf-8') as city_file: with open(city_filename, 'r', encoding='utf-8') as city_file:
city = city_file.read().replace('\n', '') city1 = city_file.read()
city = remove_eol(city1)
locations_filename = base_dir + '/custom_locations.txt' locations_filename = base_dir + '/custom_locations.txt'
if not os.path.isfile(locations_filename): if not os.path.isfile(locations_filename):
locations_filename = base_dir + '/locations.txt' locations_filename = base_dir + '/locations.txt'

3
webapp_timeline.py
View File

@ -19,6 +19,7 @@ from utils import remove_id_ending
from utils import acct_dir from utils import acct_dir
from utils import is_float from utils import is_float
from utils import local_actor_url from utils import local_actor_url
from utils import remove_eol
from follow import follower_approval_active from follow import follower_approval_active
from person import is_person_snoozed from person import is_person_snoozed
from markdown import markdown_to_html from markdown import markdown_to_html
@ -514,7 +515,7 @@ def html_timeline(css_cache: {}, default_timeline: str,
calendar_image = 'calendar_notify.png' calendar_image = 'calendar_notify.png'
with open(calendar_file, 'r', encoding='utf-8') as calfile: with open(calendar_file, 'r', encoding='utf-8') as calfile:
calendar_path = calfile.read().replace('##sent##', '') calendar_path = calfile.read().replace('##sent##', '')
calendar_path = calendar_path.replace('\n', '').replace('\r', '') calendar_path = remove_eol(calendar_path)
if '/calendar' not in calendar_path: if '/calendar' not in calendar_path:
calendar_path = '/calendar' calendar_path = '/calendar'

4
webapp_utils.py
View File

@ -28,6 +28,7 @@ from utils import get_video_extensions
from utils import get_image_extensions from utils import get_image_extensions
from utils import local_actor_url from utils import local_actor_url
from utils import text_in_file from utils import text_in_file
from utils import remove_eol
from cache import store_person_in_cache from cache import store_person_in_cache
from content import add_html_tags from content import add_html_tags
from content import replace_emoji_from_tags from content import replace_emoji_from_tags
@ -1822,7 +1823,8 @@ def html_common_emoji(base_dir: str, no_of_emoji: int) -> str:
ctr = 0 ctr = 0
html_str = '' html_str = ''
while ctr < no_of_emoji and line_ctr < len(common_emoji): while ctr < no_of_emoji and line_ctr < len(common_emoji):
emoji_name = common_emoji[line_ctr].split(' ')[1].replace('\n', '') emoji_name1 = common_emoji[line_ctr].split(' ')[1]
emoji_name = remove_eol(emoji_name1)
emoji_icon_name = emoji_name emoji_icon_name = emoji_name
emoji_filename = base_dir + '/emoji/' + emoji_name + '.png' emoji_filename = base_dir + '/emoji/' + emoji_name + '.png'
if not os.path.isfile(emoji_filename): if not os.path.isfile(emoji_filename):