Merge branch 'main' of gitlab.com:bashrc2/epicyon

2022-07-21 12:45:19 +01:00 · 2022-07-21 12:45:19 +01:00 · 8d2618f0f8
parent 91190e2cd0 9cb863a185
commit 8d2618f0f8
30 changed files with 325 additions and 32 deletions
--- a/content.py
+++ b/content.py
@ -1359,6 +1359,16 @@ def extract_media_in_form_post(post_bytes, boundary, name: str):
    return media_bytes, post_bytes[:image_start_location] + remainder


+def _valid_follows_csv(content: str) -> bool:
+    """is the given content a valid csv file containing imported follows?
+    """
+    if ',' not in content:
+        return False
+    if 'Account address,' not in content:
+        return False
+    return True
+
+
 def save_media_in_form_post(media_bytes, debug: bool,
                            filename_base: str = None) -> (str, str):
    """Saves the given media bytes extracted from http form POST
@ -1396,7 +1406,7 @@ def save_media_in_form_post(media_bytes, debug: bool,
    filename = None

    # directly search the binary array for the beginning
-    # of an image
+    # of an image, zip or csv
    extension_list = {
        'png': 'image/png',
        'jpeg': 'image/jpeg',
@ -1411,7 +1421,9 @@ def save_media_in_form_post(media_bytes, debug: bool,
        'ogg': 'audio/ogg',
        'opus': 'audio/opus',
        'flac': 'audio/flac',
-        'zip': 'application/zip'
+        'zip': 'application/zip',
+        'csv': 'text/csv',
+        'csv2': 'text/plain'
    }
    detected_extension = None
    for extension, content_type in extension_list.items():
@ -1423,6 +1435,8 @@ def save_media_in_form_post(media_bytes, debug: bool,
                extension = 'jpg'
            elif extension == 'mpeg':
                extension = 'mp3'
+            elif extension == 'csv2':
+                extension = 'csv'
            if filename_base:
                filename = filename_base + '.' + extension
            search_lst = search_str.decode().split('/', maxsplit=1)
@ -1468,6 +1482,11 @@ def save_media_in_form_post(media_bytes, debug: bool,
        svg_str = svg_str.decode()
        if dangerous_svg(svg_str, False):
            return None, None
+    elif detected_extension == 'csv':
+        csv_str = media_bytes[start_pos:]
+        csv_str = csv_str.decode()
+        if not _valid_follows_csv(csv_str):
+            return None, None

    try:
        with open(filename, 'wb') as fp_media:
--- a/daemon.py
+++ b/daemon.py
@ -397,6 +397,7 @@ from crawlers import update_known_crawlers
 from crawlers import blocked_user_agent
 from crawlers import load_known_web_bots
 from qrcode import save_domain_qrcode
+from importFollowing import run_import_following_watchdog
 import os


@ -5504,6 +5505,7 @@ class PubServer(BaseHTTPRequestHandler):
                'banner', 'search_banner',
                'instanceLogo',
                'left_col_image', 'right_col_image',
+                'submitImportFollows',
                'submitImportTheme'
            )
            profile_media_types_uploaded = {}
@ -5517,18 +5519,18 @@ class PubServer(BaseHTTPRequestHandler):

                if debug:
                    print('DEBUG: profile update extracting ' + m_type +
-                          ' image, zip or font from POST')
+                          ' image, zip, csv or font from POST')
                media_bytes, post_bytes = \
                    extract_media_in_form_post(post_bytes, boundary, m_type)
                if media_bytes:
                    if debug:
                        print('DEBUG: profile update ' + m_type +
-                              ' image, zip or font was found. ' +
+                              ' image, zip, csv or font was found. ' +
                              str(len(media_bytes)) + ' bytes')
                else:
                    if debug:
                        print('DEBUG: profile update, no ' + m_type +
-                              ' image, zip or font was found in POST')
+                              ' image, zip, csv or font was found in POST')
                    continue

                # Note: a .temp extension is used here so that at no
@ -5548,6 +5550,10 @@ class PubServer(BaseHTTPRequestHandler):
                        except OSError:
                            print('EX: _profile_edit unable to delete ' +
                                  filename_base)
+                elif m_type == 'submitImportFollows':
+                    filename_base = \
+                        acct_dir(base_dir, nickname, domain) + \
+                        '/import_following.csv'
                else:
                    filename_base = \
                        acct_dir(base_dir, nickname, domain) + \
@ -5558,10 +5564,18 @@ class PubServer(BaseHTTPRequestHandler):
                                            filename_base)
                if filename:
                    print('Profile update POST ' + m_type +
-                          ' media, zip or font filename is ' + filename)
+                          ' media, zip, csv or font filename is ' + filename)
                else:
                    print('Profile update, no ' + m_type +
-                          ' media, zip or font filename in POST')
+                          ' media, zip, csv or font filename in POST')
+                    continue
+
+                if m_type == 'submitImportFollows':
+                    if os.path.isfile(filename_base):
+                        print(nickname + ' imported follows csv')
+                    else:
+                        print('WARN: failed to import follows from csv for ' +
+                              nickname)
                    continue

                if m_type == 'submitImportTheme':
@ -21623,6 +21637,12 @@ def run_daemon(preferred_podcast_formats: [],
    httpd.thrCheckActor = {}

    if not unit_test:
+        print('THREAD: Creating import following watchdog')
+        httpd.thrImportFollowing = \
+            thread_with_trace(target=run_import_following_watchdog,
+                              args=(project_version, httpd), daemon=True)
+        httpd.thrImportFollowing.start()
+
        print('THREAD: Creating inbox queue watchdog')
        httpd.thrWatchdog = \
            thread_with_trace(target=run_inbox_queue_watchdog,
--- a/importFollowing.py
+++ b/importFollowing.py
@ -0,0 +1,219 @@
+__filename__ = "importFollowing.py"
+__author__ = "Bob Mottram"
+__license__ = "AGPL3+"
+__version__ = "1.3.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@libreserver.org"
+__status__ = "Production"
+__module_group__ = "Core"
+
+import os
+import time
+import random
+from utils import is_account_dir
+from utils import get_nickname_from_actor
+from utils import get_domain_from_actor
+from follow import is_following_actor
+from follow import send_follow_request
+from session import create_session
+from session import set_session_for_sender
+
+
+def _establish_import_session(httpd,
+                              calling_function: str,
+                              curr_session,
+                              proxy_type: str):
+    """Recreates session if needed
+    """
+    if curr_session:
+        return curr_session
+    print('DEBUG: creating new import session during ' + calling_function)
+    curr_session = create_session(proxy_type)
+    if curr_session:
+        set_session_for_sender(httpd, proxy_type, curr_session)
+        return curr_session
+    print('ERROR: failed to create import session during ' +
+          calling_function)
+    return None
+
+
+def _update_import_following(base_dir: str,
+                             handle: str, httpd,
+                             import_filename: str) -> bool:
+    """Send out follow requests from the import csv file
+    """
+    following_str = ''
+    try:
+        with open(import_filename, 'r', encoding='utf-8') as fp_import:
+            following_str = fp_import.read()
+    except OSError:
+        print('Ex: failed to load import file ' + import_filename)
+        return False
+    if following_str:
+        main_session = None
+        lines = following_str.split('\n')
+        random.shuffle(lines)
+        nickname = handle.split('@')[0]
+        domain = handle.split('@')[1]
+        for line in lines:
+            orig_line = line
+            line = line.strip()
+            if ',' in line:
+                line = line.split(',')[0].strip()
+            if line.startswith('#'):
+                continue
+            following_nickname = get_nickname_from_actor(line)
+            if not following_nickname:
+                continue
+            following_domain, following_port = get_domain_from_actor(line)
+            if not following_domain:
+                continue
+            if following_nickname == nickname and \
+               following_domain == domain:
+                # don't follow yourself
+                continue
+            following_handle = following_nickname + '@' + following_domain
+            if is_following_actor(base_dir,
+                                  nickname, domain, following_handle):
+                # remove the followed handle from the import list
+                following_str = following_str.replace(orig_line + '\n', '')
+                try:
+                    with open(import_filename, 'w+',
+                              encoding='utf-8') as fp_import:
+                        fp_import.write(following_str)
+                except OSError:
+                    print('EX: unable to remove import 1 ' + line +
+                          ' from ' + import_filename)
+                continue
+
+            # send follow request
+            curr_domain = domain
+            curr_port = httpd.port
+            curr_http_prefix = httpd.http_prefix
+            following_actor = following_handle
+            if ':' in following_domain:
+                following_port = following_handle.split(':')[1]
+                following_domain = following_domain.split(':')[0]
+
+            # get the appropriate session
+            curr_session = main_session
+            curr_proxy_type = httpd.proxy_type
+            use_onion_session = False
+            use_i2p_session = False
+            if '.onion' not in domain and \
+               httpd.onion_domain and '.onion' in following_domain:
+                curr_session = httpd.session_onion
+                curr_domain = httpd.onion_domain
+                curr_port = 80
+                following_port = 80
+                curr_http_prefix = 'http'
+                curr_proxy_type = 'tor'
+                use_onion_session = True
+            if '.i2p' not in domain and \
+               httpd.i2p_domain and '.i2p' in following_domain:
+                curr_session = httpd.session_i2p
+                curr_domain = httpd.i2p_domain
+                curr_port = 80
+                following_port = 80
+                curr_http_prefix = 'http'
+                curr_proxy_type = 'i2p'
+                use_i2p_session = True
+
+            curr_session = \
+                _establish_import_session(httpd, "import follow",
+                                          curr_session, curr_proxy_type)
+            if curr_session:
+                if use_onion_session:
+                    httpd.session_onion = curr_session
+                elif use_i2p_session:
+                    httpd.session_i2p = curr_session
+                else:
+                    main_session = curr_session
+
+            send_follow_request(curr_session,
+                                base_dir, nickname,
+                                domain, curr_domain, curr_port,
+                                curr_http_prefix,
+                                following_nickname,
+                                following_domain,
+                                following_actor,
+                                following_port, curr_http_prefix,
+                                False, httpd.federation_list,
+                                httpd.send_threads,
+                                httpd.postLog,
+                                httpd.cached_webfingers,
+                                httpd.person_cache, httpd.debug,
+                                httpd.project_version,
+                                httpd.signing_priv_key_pem,
+                                httpd.domain,
+                                httpd.onion_domain,
+                                httpd.i2p_domain)
+
+            # remove the followed handle from the import list
+            following_str = following_str.replace(orig_line + '\n', '')
+            try:
+                with open(import_filename, 'w+',
+                          encoding='utf-8') as fp_import:
+                    fp_import.write(following_str)
+            except OSError:
+                print('EX: unable to remove import 2 ' + line +
+                      ' from ' + import_filename)
+            return True
+    return False
+
+
+def run_import_following(base_dir: str, httpd):
+    """Sends out follow requests for imported following csv files
+    """
+    while True:
+        time.sleep(20)
+
+        # get a list of accounts on the instance, in random sequence
+        accounts_list = []
+        for _, dirs, _ in os.walk(base_dir + '/accounts'):
+            for account in dirs:
+                if '@' not in account:
+                    continue
+                if not is_account_dir(account):
+                    continue
+                accounts_list.append(account)
+            break
+        if not accounts_list:
+            continue
+
+        # check if each accounts has an import csv
+        random.shuffle(accounts_list)
+        for account in accounts_list:
+            account_dir = base_dir + '/accounts/' + account
+            import_filename = account_dir + '/import_following.csv'
+
+            if not os.path.isfile(import_filename):
+                continue
+            if not _update_import_following(base_dir, account, httpd,
+                                            import_filename):
+                try:
+                    os.remove(import_filename)
+                except OSError:
+                    print('EX: unable to remove import file ' +
+                          import_filename)
+            else:
+                break
+
+
+def run_import_following_watchdog(project_version: str, httpd) -> None:
+    """Imports following lists from csv for every account on the instance
+    """
+    print('THREAD: Starting import following watchdog ' + project_version)
+    import_following_original = \
+        httpd.thrImportFollowing.clone(run_import_following)
+    httpd.thrImportFollowing.start()
+    while True:
+        time.sleep(20)
+        if httpd.thrImportFollowing.is_alive():
+            continue
+        httpd.thrImportFollowing.kill()
+        print('THREAD: restarting import following watchdog')
+        httpd.thrImportFollowing = \
+            import_following_original.clone(run_import_following)
+        httpd.thrImportFollowing.start()
+        print('Restarting import following...')
--- a/tests.py
+++ b/tests.py
@ -5408,6 +5408,8 @@ def _test_functions():
        'get_document_loader',
        'run_inbox_queue_watchdog',
        'run_inbox_queue',
+        'run_import_following',
+        'run_import_following_watchdog',
        'run_post_schedule',
        'run_post_schedule_watchdog',
        'str2bool',
--- a/translations/ar.json
+++ b/translations/ar.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "ستتم إضافة تحذيرات المحتوى لما يلي",
    "nowplaying": "الان العب",
    "NowPlaying": "الان العب",
-    "Import and Export": "استيراد وتصدير"
+    "Import and Export": "استيراد وتصدير",
+    "Import Follows": "يتبع الاستيراد"
 }
--- a/translations/bn.json
+++ b/translations/bn.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "নিম্নলিখিত জন্য বিষয়বস্তু সতর্কতা যোগ করা হবে",
    "nowplaying": "এখন চলছে",
    "NowPlaying": "এখন চলছে",
-    "Import and Export": "আমদানি এবং রপ্তানি"
+    "Import and Export": "আমদানি এবং রপ্তানি",
+    "Import Follows": "আমদানি অনুসরণ করে"
 }
--- a/translations/ca.json
+++ b/translations/ca.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "S'afegiran advertències de contingut per al següent",
    "nowplaying": "arajugant",
    "NowPlaying": "AraJugant",
-    "Import and Export": "Importació i Exportació"
+    "Import and Export": "Importació i Exportació",
+    "Import Follows": "Segueix la importació"
 }
--- a/translations/cy.json
+++ b/translations/cy.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Bydd rhybuddion cynnwys yn cael eu hychwanegu ar gyfer y canlynol",
    "nowplaying": "nawrynchwarae",
    "NowPlaying": "NawrYnChwarae",
-    "Import and Export": "Mewnforio ac Allforio"
+    "Import and Export": "Mewnforio ac Allforio",
+    "Import Follows": "Mewnforio Dilyn"
 }
--- a/translations/de.json
+++ b/translations/de.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Inhaltswarnungen werden für Folgendes hinzugefügt",
    "nowplaying": "läuftgerade",
    "NowPlaying": "LäuftGerade",
-    "Import and Export": "Import und Export"
+    "Import and Export": "Import und Export",
+    "Import Follows": "Import folgt"
 }
--- a/translations/el.json
+++ b/translations/el.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Θα προστεθούν προειδοποιήσεις περιεχομένου για τα ακόλουθα",
    "nowplaying": "τώραπαίζει",
    "NowPlaying": "ΤώραΠαίζει",
-    "Import and Export": "Εισάγω και εξάγω"
+    "Import and Export": "Εισάγω και εξάγω",
+    "Import Follows": "Ακολουθεί εισαγωγή"
 }
--- a/translations/en.json
+++ b/translations/en.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Content warnings will be added for the following",
    "nowplaying": "nowplaying",
    "NowPlaying": "NowPlaying",
-    "Import and Export": "Import and Export"
+    "Import and Export": "Import and Export",
+    "Import Follows": "Import Follows"
 }
--- a/translations/es.json
+++ b/translations/es.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Se agregarán advertencias de contenido para lo siguiente",
    "nowplaying": "jugandoahora",
    "NowPlaying": "JugandoAhora",
-    "Import and Export": "Importar y exportar"
+    "Import and Export": "Importar y exportar",
+    "Import Follows": "Importar seguimientos"
 }
--- a/translations/fr.json
+++ b/translations/fr.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Des avertissements de contenu seront ajoutés pour les éléments suivants",
    "nowplaying": "lectureencours",
    "NowPlaying": "LectureEnCours",
-    "Import and Export": "Importer et exporter"
+    "Import and Export": "Importer et exporter",
+    "Import Follows": "Importer suit"
 }
--- a/translations/ga.json
+++ b/translations/ga.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Cuirfear rabhaidh ábhair leis maidir leis na nithe seo a leanas",
    "nowplaying": "anoisagimirt",
    "NowPlaying": "AnoisAgImirt",
-    "Import and Export": "Iompórtáil agus Easpórtáil"
+    "Import and Export": "Iompórtáil agus Easpórtáil",
+    "Import Follows": "Leanann Iompórtáil"
 }
--- a/translations/hi.json
+++ b/translations/hi.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "निम्नलिखित के लिए सामग्री चेतावनियां जोड़ दी जाएंगी",
    "nowplaying": "अब खेल रहे हैं",
    "NowPlaying": "अब खेल रहे हैं",
-    "Import and Export": "आयात और निर्यात"
+    "Import and Export": "आयात और निर्यात",
+    "Import Follows": "आयात का अनुसरण करता है"
 }
--- a/translations/it.json
+++ b/translations/it.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Verranno aggiunti avvisi sui contenuti per quanto segue",
    "nowplaying": "ora giocando",
    "NowPlaying": "OraGiocando",
-    "Import and Export": "Importazione e esportazione"
+    "Import and Export": "Importazione e esportazione",
+    "Import Follows": "Importa segue"
 }
--- a/translations/ja.json
+++ b/translations/ja.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "以下のコンテンツ警告が追加されます",
    "nowplaying": "再生中",
    "NowPlaying": "再生中",
-    "Import and Export": "インポートとエクスポート"
+    "Import and Export": "インポートとエクスポート",
+    "Import Follows": "インポートフォロー"
 }
--- a/translations/ko.json
+++ b/translations/ko.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "다음에 대한 콘텐츠 경고가 추가됩니다.",
    "nowplaying": "지금 재생",
    "NowPlaying": "지금 재생",
-    "Import and Export": "가져오기 및 내보내기"
+    "Import and Export": "가져오기 및 내보내기",
+    "Import Follows": "가져오기 팔로우"
 }
--- a/translations/ku.json
+++ b/translations/ku.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Hişyariyên naverokê dê ji bo jêrîn werin zêdekirin",
    "nowplaying": "nihadilîze",
    "NowPlaying": "NihaDilîze",
-    "Import and Export": "Import û Export"
+    "Import and Export": "Import û Export",
+    "Import Follows": "Import Follows"
 }
--- a/translations/nl.json
+++ b/translations/nl.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Er worden inhoudswaarschuwingen toegevoegd voor het volgende:",
    "nowplaying": "nuaanhetspelen",
    "NowPlaying": "NuAanHetSpelen",
-    "Import and Export": "Importeren en exporteren"
+    "Import and Export": "Importeren en exporteren",
+    "Import Follows": "Volgt importeren"
 }
--- a/translations/oc.json
+++ b/translations/oc.json
@ -566,5 +566,6 @@
    "Content warnings will be added for the following": "Content warnings will be added for the following",
    "nowplaying": "nowplaying",
    "NowPlaying": "NowPlaying",
-    "Import and Export": "Import and Export"
+    "Import and Export": "Import and Export",
+    "Import Follows": "Import Follows"
 }
--- a/translations/pl.json
+++ b/translations/pl.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Ostrzeżenia dotyczące treści zostaną dodane do następujących",
    "nowplaying": "terazgra",
    "NowPlaying": "TerazGra",
-    "Import and Export": "Importuj i eksportuj"
+    "Import and Export": "Importuj i eksportuj",
+    "Import Follows": "Importuj obserwuje"
 }
--- a/translations/pt.json
+++ b/translations/pt.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Avisos de conteúdo serão adicionados para os seguintes",
    "nowplaying": "agorajogando",
    "NowPlaying": "AgoraJogando",
-    "Import and Export": "Importar e exportar"
+    "Import and Export": "Importar e exportar",
+    "Import Follows": "Importar seguidores"
 }
--- a/translations/ru.json
+++ b/translations/ru.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Предупреждения о содержании будут добавлены для следующих",
    "nowplaying": "сейчасиграет",
    "NowPlaying": "СейчасИграет",
-    "Import and Export": "Импорт и экспорт"
+    "Import and Export": "Импорт и экспорт",
+    "Import Follows": "Импорт подписок"
 }
--- a/translations/sw.json
+++ b/translations/sw.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Maonyo ya maudhui yataongezwa kwa yafuatayo",
    "nowplaying": "inachezasasa",
    "NowPlaying": "InachezaSasa",
-    "Import and Export": "Ingiza na Hamisha"
+    "Import and Export": "Ingiza na Hamisha",
+    "Import Follows": "Ingiza Inafuata"
 }
--- a/translations/tr.json
+++ b/translations/tr.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Aşağıdakiler için içerik uyarıları eklenecek",
    "nowplaying": "şimdioynuyor",
    "NowPlaying": "ŞimdiOynuyor",
-    "Import and Export": "İthalat ve ihracat"
+    "Import and Export": "İthalat ve ihracat",
+    "Import Follows": "Takipleri İçe Aktar"
 }
--- a/translations/uk.json
+++ b/translations/uk.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "Попередження про вміст буде додано для наступних",
    "nowplaying": "заразграє",
    "NowPlaying": "ЗаразГрає",
-    "Import and Export": "Імпорт та експорт"
+    "Import and Export": "Імпорт та експорт",
+    "Import Follows": "Імпорт слідує"
 }
--- a/translations/yi.json
+++ b/translations/yi.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "אינהאַלט וואָרנינגז וועט זיין מוסיף פֿאַר די פאלגענדע",
    "nowplaying": "איצט פּלייַינג",
    "NowPlaying": "איצט פּלייַינג",
-    "Import and Export": "אַרייַנפיר און עקספּאָרט"
+    "Import and Export": "אַרייַנפיר און עקספּאָרט",
+    "Import Follows": "אַרייַנפיר גייט"
 }
--- a/translations/zh.json
+++ b/translations/zh.json
@ -570,5 +570,6 @@
    "Content warnings will be added for the following": "将为以下内容添加内容警告",
    "nowplaying": "现在玩",
    "NowPlaying": "现在玩",
-    "Import and Export": "进出口"
+    "Import and Export": "进出口",
+    "Import Follows": "导入关注"
 }
--- a/webapp_profile.py
+++ b/webapp_profile.py
@ -2003,6 +2003,14 @@ def _html_edit_profile_import_export(nickname: str, domain: str,
    """Contact Information section of edit profile screen
    """
    edit_profile_form = begin_edit_section(translate['Import and Export'])
+
+    edit_profile_form += \
+        '<p><label class="labels">' + \
+        translate['Import Follows'] + '</label>\n'
+    edit_profile_form += '<input type="file" id="import_follows" '
+    edit_profile_form += 'name="submitImportFollows" '
+    edit_profile_form += 'accept=".csv"></p>\n'
+
    edit_profile_form += \
        '<p><a href="/users/' + nickname + \
        '/followingaccounts"><label class="labels">' + \