Automatic follow requests from imported folloing csv files

daemon.py

@@ -397,6 +397,7 @@ from crawlers import update_known_crawlers
 from crawlers import blocked_user_agent
 from crawlers import load_known_web_bots
 from qrcode import save_domain_qrcode
+from importFollowing import run_import_following_watchdog
 import os
 
 
@@ -21623,6 +21624,12 @@ def run_daemon(preferred_podcast_formats: [],
     httpd.thrCheckActor = {}
 
     if not unit_test:
+        print('THREAD: Creating import following watchdog')
+        httpd.thrImportFollowing = \
+            thread_with_trace(target=run_import_following_watchdog,
+                              args=(project_version, httpd), daemon=True)
+        httpd.thrImportFollowing.start()
+
         print('THREAD: Creating inbox queue watchdog')
         httpd.thrWatchdog = \
             thread_with_trace(target=run_inbox_queue_watchdog,

importFollowing.py

@@ -0,0 +1,212 @@
+__filename__ = "importFollowing.py"
+__author__ = "Bob Mottram"
+__license__ = "AGPL3+"
+__version__ = "1.3.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@libreserver.org"
+__status__ = "Production"
+__module_group__ = "Core"
+
+import os
+import time
+import random
+from utils import is_account_dir
+from follow import is_following_actor
+from follow import send_follow_request
+from session import create_session
+from session import set_session_for_sender
+
+
+def _establish_import_session(httpd,
+                              calling_function: str,
+                              curr_session,
+                              proxy_type: str):
+    """Recreates session if needed
+    """
+    if curr_session:
+        return curr_session
+    print('DEBUG: creating new import session during ' + calling_function)
+    curr_session = create_session(proxy_type)
+    if curr_session:
+        set_session_for_sender(httpd, proxy_type, curr_session)
+        return curr_session
+    print('ERROR: failed to create import session during ' +
+          calling_function)
+    return None
+
+
+def _update_import_following(base_dir: str,
+                             handle: str, httpd,
+                             import_filename: str) -> bool:
+    """Send out follow requests from the import csv file
+    """
+    following_str = ''
+    try:
+        with open(import_filename, 'r', encoding='utf-8') as fp_import:
+            following_str = fp_import.read()
+    except OSError:
+        print('Ex: failed to load import file ' + import_filename)
+        return False
+    if following_str:
+        main_session = None
+        lines = following_str.split('\n')
+        random.shuffle(lines)
+        nickname = handle.split('@')[0]
+        domain = handle.split('@')[1]
+        for line in lines:
+            orig_line = line
+            line = line.strip()
+            if ',' in line:
+                line = line.split(',')[0].strip()
+            if line.startswith('#'):
+                continue
+            if '@' not in line:
+                continue
+            following_handle = line
+            if is_following_actor(base_dir,
+                                  nickname, domain, following_handle):
+                # remove the followed handle from the import list
+                following_str = following_str.replace(orig_line + '\n', '')
+                try:
+                    with open(import_filename, 'w+',
+                              encoding='utf-8') as fp_import:
+                        fp_import.write(following_str)
+                except OSError:
+                    print('EX: unable to remove import 1 ' + line +
+                          ' from ' + import_filename)
+                continue
+
+            # send follow request
+            curr_domain = domain
+            curr_port = httpd.port
+            curr_http_prefix = httpd.http_prefix
+            following_nickname = following_handle.split('@')[0]
+            following_domain = following_handle.split('@')[1]
+            following_actor = following_handle
+            following_port = httpd.port
+            if ':' in following_domain:
+                following_port = following_handle.split(':')[1]
+                following_domain = following_domain.split(':')[0]
+
+            # get the appropriate session
+            curr_session = main_session
+            curr_proxy_type = httpd.proxy_type
+            use_onion_session = False
+            use_i2p_session = False
+            if '.onion' not in domain and \
+               httpd.onion_domain and '.onion' in following_domain:
+                curr_session = httpd.session_onion
+                curr_domain = httpd.onion_domain
+                curr_port = 80
+                following_port = 80
+                curr_http_prefix = 'http'
+                curr_proxy_type = 'tor'
+                use_onion_session = True
+            if '.i2p' not in domain and \
+               httpd.i2p_domain and '.i2p' in following_domain:
+                curr_session = httpd.session_i2p
+                curr_domain = httpd.i2p_domain
+                curr_port = 80
+                following_port = 80
+                curr_http_prefix = 'http'
+                curr_proxy_type = 'i2p'
+                use_i2p_session = True
+
+            curr_session = \
+                _establish_import_session(httpd, "import follow",
+                                          curr_session, curr_proxy_type)
+            if curr_session:
+                if use_onion_session:
+                    httpd.session_onion = curr_session
+                elif use_i2p_session:
+                    httpd.session_i2p = curr_session
+                else:
+                    main_session = curr_session
+
+            send_follow_request(curr_session,
+                                base_dir, nickname,
+                                domain, curr_domain, curr_port,
+                                curr_http_prefix,
+                                following_nickname,
+                                following_domain,
+                                following_actor,
+                                following_port, curr_http_prefix,
+                                False, httpd.federation_list,
+                                httpd.send_threads,
+                                httpd.postLog,
+                                httpd.cached_webfingers,
+                                httpd.person_cache, httpd.debug,
+                                httpd.project_version,
+                                httpd.signing_priv_key_pem,
+                                httpd.domain,
+                                httpd.onion_domain,
+                                httpd.i2p_domain)
+
+            # remove the followed handle from the import list
+            following_str = following_str.replace(orig_line + '\n', '')
+            try:
+                with open(import_filename, 'w+',
+                          encoding='utf-8') as fp_import:
+                    fp_import.write(following_str)
+            except OSError:
+                print('EX: unable to remove import 2 ' + line +
+                      ' from ' + import_filename)
+            return True
+    return False
+
+
+def run_import_following(base_dir: str, httpd):
+    """Sends out follow requests for imported following csv files
+    """
+    while True:
+        time.sleep(10)
+
+        # get a list of accounts on the instance, in random sequence
+        accounts_list = []
+        for _, dirs, _ in os.walk(base_dir + '/accounts'):
+            for account in dirs:
+                if '@' not in account:
+                    continue
+                if not is_account_dir(account):
+                    continue
+                accounts_list.append(account)
+            break
+        if not accounts_list:
+            continue
+
+        # check if each accounts has an import csv
+        random.shuffle(accounts_list)
+        for account in accounts_list:
+            account_dir = base_dir + '/accounts/' + account
+            import_filename = account_dir + '/import_following.csv'
+
+            if not os.path.isfile(import_filename):
+                continue
+            if not _update_import_following(base_dir, account, httpd,
+                                            import_filename):
+                try:
+                    os.remove(import_filename)
+                except OSError:
+                    print('EX: unable to remove import file ' +
+                          import_filename)
+            else:
+                break
+
+
+def run_import_following_watchdog(project_version: str, httpd) -> None:
+    """Imports following lists from csv for every account on the instance
+    """
+    print('THREAD: Starting import following watchdog ' + project_version)
+    import_following_original = \
+        httpd.thrImportFollowing.clone(run_import_following)
+    httpd.thrImportFollowing.start()
+    while True:
+        time.sleep(20)
+        if httpd.thrImportFollowing.is_alive():
+            continue
+        httpd.thrImportFollowing.kill()
+        print('THREAD: restarting import following watchdog')
+        httpd.thrImportFollowing = \
+            import_following_original.clone(run_import_following)
+        httpd.thrImportFollowing.start()
+        print('Restarting import following...')

tests.py

@@ -5408,6 +5408,8 @@ def _test_functions():
         'get_document_loader',
         'run_inbox_queue_watchdog',
         'run_inbox_queue',
+        'run_import_following',
+        'run_import_following_watchdog',
         'run_post_schedule',
         'run_post_schedule_watchdog',
         'str2bool',