From ffd7ce2476e9dedf04e51a71ef6cc183d36514df Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 20 Jul 2022 23:47:08 +0100 Subject: [PATCH] Automatic follow requests from imported folloing csv files --- daemon.py | 7 ++ importFollowing.py | 212 +++++++++++++++++++++++++++++++++++++++++++++ tests.py | 2 + 3 files changed, 221 insertions(+) create mode 100644 importFollowing.py diff --git a/daemon.py b/daemon.py index 9acff8c2e..686866010 100644 --- a/daemon.py +++ b/daemon.py @@ -397,6 +397,7 @@ from crawlers import update_known_crawlers from crawlers import blocked_user_agent from crawlers import load_known_web_bots from qrcode import save_domain_qrcode +from importFollowing import run_import_following_watchdog import os @@ -21623,6 +21624,12 @@ def run_daemon(preferred_podcast_formats: [], httpd.thrCheckActor = {} if not unit_test: + print('THREAD: Creating import following watchdog') + httpd.thrImportFollowing = \ + thread_with_trace(target=run_import_following_watchdog, + args=(project_version, httpd), daemon=True) + httpd.thrImportFollowing.start() + print('THREAD: Creating inbox queue watchdog') httpd.thrWatchdog = \ thread_with_trace(target=run_inbox_queue_watchdog, diff --git a/importFollowing.py b/importFollowing.py new file mode 100644 index 000000000..3878f9d1a --- /dev/null +++ b/importFollowing.py @@ -0,0 +1,212 @@ +__filename__ = "importFollowing.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.3.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@libreserver.org" +__status__ = "Production" +__module_group__ = "Core" + +import os +import time +import random +from utils import is_account_dir +from follow import is_following_actor +from follow import send_follow_request +from session import create_session +from session import set_session_for_sender + + +def _establish_import_session(httpd, + calling_function: str, + curr_session, + proxy_type: str): + """Recreates session if needed + """ + if curr_session: + return curr_session + print('DEBUG: creating new import session during ' + calling_function) + curr_session = create_session(proxy_type) + if curr_session: + set_session_for_sender(httpd, proxy_type, curr_session) + return curr_session + print('ERROR: failed to create import session during ' + + calling_function) + return None + + +def _update_import_following(base_dir: str, + handle: str, httpd, + import_filename: str) -> bool: + """Send out follow requests from the import csv file + """ + following_str = '' + try: + with open(import_filename, 'r', encoding='utf-8') as fp_import: + following_str = fp_import.read() + except OSError: + print('Ex: failed to load import file ' + import_filename) + return False + if following_str: + main_session = None + lines = following_str.split('\n') + random.shuffle(lines) + nickname = handle.split('@')[0] + domain = handle.split('@')[1] + for line in lines: + orig_line = line + line = line.strip() + if ',' in line: + line = line.split(',')[0].strip() + if line.startswith('#'): + continue + if '@' not in line: + continue + following_handle = line + if is_following_actor(base_dir, + nickname, domain, following_handle): + # remove the followed handle from the import list + following_str = following_str.replace(orig_line + '\n', '') + try: + with open(import_filename, 'w+', + encoding='utf-8') as fp_import: + fp_import.write(following_str) + except OSError: + print('EX: unable to remove import 1 ' + line + + ' from ' + import_filename) + continue + + # send follow request + curr_domain = domain + curr_port = httpd.port + curr_http_prefix = httpd.http_prefix + following_nickname = following_handle.split('@')[0] + following_domain = following_handle.split('@')[1] + following_actor = following_handle + following_port = httpd.port + if ':' in following_domain: + following_port = following_handle.split(':')[1] + following_domain = following_domain.split(':')[0] + + # get the appropriate session + curr_session = main_session + curr_proxy_type = httpd.proxy_type + use_onion_session = False + use_i2p_session = False + if '.onion' not in domain and \ + httpd.onion_domain and '.onion' in following_domain: + curr_session = httpd.session_onion + curr_domain = httpd.onion_domain + curr_port = 80 + following_port = 80 + curr_http_prefix = 'http' + curr_proxy_type = 'tor' + use_onion_session = True + if '.i2p' not in domain and \ + httpd.i2p_domain and '.i2p' in following_domain: + curr_session = httpd.session_i2p + curr_domain = httpd.i2p_domain + curr_port = 80 + following_port = 80 + curr_http_prefix = 'http' + curr_proxy_type = 'i2p' + use_i2p_session = True + + curr_session = \ + _establish_import_session(httpd, "import follow", + curr_session, curr_proxy_type) + if curr_session: + if use_onion_session: + httpd.session_onion = curr_session + elif use_i2p_session: + httpd.session_i2p = curr_session + else: + main_session = curr_session + + send_follow_request(curr_session, + base_dir, nickname, + domain, curr_domain, curr_port, + curr_http_prefix, + following_nickname, + following_domain, + following_actor, + following_port, curr_http_prefix, + False, httpd.federation_list, + httpd.send_threads, + httpd.postLog, + httpd.cached_webfingers, + httpd.person_cache, httpd.debug, + httpd.project_version, + httpd.signing_priv_key_pem, + httpd.domain, + httpd.onion_domain, + httpd.i2p_domain) + + # remove the followed handle from the import list + following_str = following_str.replace(orig_line + '\n', '') + try: + with open(import_filename, 'w+', + encoding='utf-8') as fp_import: + fp_import.write(following_str) + except OSError: + print('EX: unable to remove import 2 ' + line + + ' from ' + import_filename) + return True + return False + + +def run_import_following(base_dir: str, httpd): + """Sends out follow requests for imported following csv files + """ + while True: + time.sleep(10) + + # get a list of accounts on the instance, in random sequence + accounts_list = [] + for _, dirs, _ in os.walk(base_dir + '/accounts'): + for account in dirs: + if '@' not in account: + continue + if not is_account_dir(account): + continue + accounts_list.append(account) + break + if not accounts_list: + continue + + # check if each accounts has an import csv + random.shuffle(accounts_list) + for account in accounts_list: + account_dir = base_dir + '/accounts/' + account + import_filename = account_dir + '/import_following.csv' + + if not os.path.isfile(import_filename): + continue + if not _update_import_following(base_dir, account, httpd, + import_filename): + try: + os.remove(import_filename) + except OSError: + print('EX: unable to remove import file ' + + import_filename) + else: + break + + +def run_import_following_watchdog(project_version: str, httpd) -> None: + """Imports following lists from csv for every account on the instance + """ + print('THREAD: Starting import following watchdog ' + project_version) + import_following_original = \ + httpd.thrImportFollowing.clone(run_import_following) + httpd.thrImportFollowing.start() + while True: + time.sleep(20) + if httpd.thrImportFollowing.is_alive(): + continue + httpd.thrImportFollowing.kill() + print('THREAD: restarting import following watchdog') + httpd.thrImportFollowing = \ + import_following_original.clone(run_import_following) + httpd.thrImportFollowing.start() + print('Restarting import following...') diff --git a/tests.py b/tests.py index 005e26467..78444e709 100644 --- a/tests.py +++ b/tests.py @@ -5408,6 +5408,8 @@ def _test_functions(): 'get_document_loader', 'run_inbox_queue_watchdog', 'run_inbox_queue', + 'run_import_following', + 'run_import_following_watchdog', 'run_post_schedule', 'run_post_schedule_watchdog', 'str2bool',