diff --git a/content.py b/content.py index 29aa60d4b..9ef638d44 100644 --- a/content.py +++ b/content.py @@ -1359,6 +1359,16 @@ def extract_media_in_form_post(post_bytes, boundary, name: str): return media_bytes, post_bytes[:image_start_location] + remainder +def _valid_follows_csv(content: str) -> bool: + """is the given content a valid csv file containing imported follows? + """ + if ',' not in content: + return False + if 'Account address,' not in content: + return False + return True + + def save_media_in_form_post(media_bytes, debug: bool, filename_base: str = None) -> (str, str): """Saves the given media bytes extracted from http form POST @@ -1396,7 +1406,7 @@ def save_media_in_form_post(media_bytes, debug: bool, filename = None # directly search the binary array for the beginning - # of an image + # of an image, zip or csv extension_list = { 'png': 'image/png', 'jpeg': 'image/jpeg', @@ -1411,7 +1421,9 @@ def save_media_in_form_post(media_bytes, debug: bool, 'ogg': 'audio/ogg', 'opus': 'audio/opus', 'flac': 'audio/flac', - 'zip': 'application/zip' + 'zip': 'application/zip', + 'csv': 'text/csv', + 'csv2': 'text/plain' } detected_extension = None for extension, content_type in extension_list.items(): @@ -1423,6 +1435,8 @@ def save_media_in_form_post(media_bytes, debug: bool, extension = 'jpg' elif extension == 'mpeg': extension = 'mp3' + elif extension == 'csv2': + extension = 'csv' if filename_base: filename = filename_base + '.' + extension search_lst = search_str.decode().split('/', maxsplit=1) @@ -1468,6 +1482,11 @@ def save_media_in_form_post(media_bytes, debug: bool, svg_str = svg_str.decode() if dangerous_svg(svg_str, False): return None, None + elif detected_extension == 'csv': + csv_str = media_bytes[start_pos:] + csv_str = csv_str.decode() + if not _valid_follows_csv(csv_str): + return None, None try: with open(filename, 'wb') as fp_media: diff --git a/daemon.py b/daemon.py index 9acff8c2e..70ded593f 100644 --- a/daemon.py +++ b/daemon.py @@ -397,6 +397,7 @@ from crawlers import update_known_crawlers from crawlers import blocked_user_agent from crawlers import load_known_web_bots from qrcode import save_domain_qrcode +from importFollowing import run_import_following_watchdog import os @@ -5504,6 +5505,7 @@ class PubServer(BaseHTTPRequestHandler): 'banner', 'search_banner', 'instanceLogo', 'left_col_image', 'right_col_image', + 'submitImportFollows', 'submitImportTheme' ) profile_media_types_uploaded = {} @@ -5517,18 +5519,18 @@ class PubServer(BaseHTTPRequestHandler): if debug: print('DEBUG: profile update extracting ' + m_type + - ' image, zip or font from POST') + ' image, zip, csv or font from POST') media_bytes, post_bytes = \ extract_media_in_form_post(post_bytes, boundary, m_type) if media_bytes: if debug: print('DEBUG: profile update ' + m_type + - ' image, zip or font was found. ' + + ' image, zip, csv or font was found. ' + str(len(media_bytes)) + ' bytes') else: if debug: print('DEBUG: profile update, no ' + m_type + - ' image, zip or font was found in POST') + ' image, zip, csv or font was found in POST') continue # Note: a .temp extension is used here so that at no @@ -5548,6 +5550,10 @@ class PubServer(BaseHTTPRequestHandler): except OSError: print('EX: _profile_edit unable to delete ' + filename_base) + elif m_type == 'submitImportFollows': + filename_base = \ + acct_dir(base_dir, nickname, domain) + \ + '/import_following.csv' else: filename_base = \ acct_dir(base_dir, nickname, domain) + \ @@ -5558,10 +5564,18 @@ class PubServer(BaseHTTPRequestHandler): filename_base) if filename: print('Profile update POST ' + m_type + - ' media, zip or font filename is ' + filename) + ' media, zip, csv or font filename is ' + filename) else: print('Profile update, no ' + m_type + - ' media, zip or font filename in POST') + ' media, zip, csv or font filename in POST') + continue + + if m_type == 'submitImportFollows': + if os.path.isfile(filename_base): + print(nickname + ' imported follows csv') + else: + print('WARN: failed to import follows from csv for ' + + nickname) continue if m_type == 'submitImportTheme': @@ -21623,6 +21637,12 @@ def run_daemon(preferred_podcast_formats: [], httpd.thrCheckActor = {} if not unit_test: + print('THREAD: Creating import following watchdog') + httpd.thrImportFollowing = \ + thread_with_trace(target=run_import_following_watchdog, + args=(project_version, httpd), daemon=True) + httpd.thrImportFollowing.start() + print('THREAD: Creating inbox queue watchdog') httpd.thrWatchdog = \ thread_with_trace(target=run_inbox_queue_watchdog, diff --git a/importFollowing.py b/importFollowing.py new file mode 100644 index 000000000..c81bc524d --- /dev/null +++ b/importFollowing.py @@ -0,0 +1,219 @@ +__filename__ = "importFollowing.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.3.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@libreserver.org" +__status__ = "Production" +__module_group__ = "Core" + +import os +import time +import random +from utils import is_account_dir +from utils import get_nickname_from_actor +from utils import get_domain_from_actor +from follow import is_following_actor +from follow import send_follow_request +from session import create_session +from session import set_session_for_sender + + +def _establish_import_session(httpd, + calling_function: str, + curr_session, + proxy_type: str): + """Recreates session if needed + """ + if curr_session: + return curr_session + print('DEBUG: creating new import session during ' + calling_function) + curr_session = create_session(proxy_type) + if curr_session: + set_session_for_sender(httpd, proxy_type, curr_session) + return curr_session + print('ERROR: failed to create import session during ' + + calling_function) + return None + + +def _update_import_following(base_dir: str, + handle: str, httpd, + import_filename: str) -> bool: + """Send out follow requests from the import csv file + """ + following_str = '' + try: + with open(import_filename, 'r', encoding='utf-8') as fp_import: + following_str = fp_import.read() + except OSError: + print('Ex: failed to load import file ' + import_filename) + return False + if following_str: + main_session = None + lines = following_str.split('\n') + random.shuffle(lines) + nickname = handle.split('@')[0] + domain = handle.split('@')[1] + for line in lines: + orig_line = line + line = line.strip() + if ',' in line: + line = line.split(',')[0].strip() + if line.startswith('#'): + continue + following_nickname = get_nickname_from_actor(line) + if not following_nickname: + continue + following_domain, following_port = get_domain_from_actor(line) + if not following_domain: + continue + if following_nickname == nickname and \ + following_domain == domain: + # don't follow yourself + continue + following_handle = following_nickname + '@' + following_domain + if is_following_actor(base_dir, + nickname, domain, following_handle): + # remove the followed handle from the import list + following_str = following_str.replace(orig_line + '\n', '') + try: + with open(import_filename, 'w+', + encoding='utf-8') as fp_import: + fp_import.write(following_str) + except OSError: + print('EX: unable to remove import 1 ' + line + + ' from ' + import_filename) + continue + + # send follow request + curr_domain = domain + curr_port = httpd.port + curr_http_prefix = httpd.http_prefix + following_actor = following_handle + if ':' in following_domain: + following_port = following_handle.split(':')[1] + following_domain = following_domain.split(':')[0] + + # get the appropriate session + curr_session = main_session + curr_proxy_type = httpd.proxy_type + use_onion_session = False + use_i2p_session = False + if '.onion' not in domain and \ + httpd.onion_domain and '.onion' in following_domain: + curr_session = httpd.session_onion + curr_domain = httpd.onion_domain + curr_port = 80 + following_port = 80 + curr_http_prefix = 'http' + curr_proxy_type = 'tor' + use_onion_session = True + if '.i2p' not in domain and \ + httpd.i2p_domain and '.i2p' in following_domain: + curr_session = httpd.session_i2p + curr_domain = httpd.i2p_domain + curr_port = 80 + following_port = 80 + curr_http_prefix = 'http' + curr_proxy_type = 'i2p' + use_i2p_session = True + + curr_session = \ + _establish_import_session(httpd, "import follow", + curr_session, curr_proxy_type) + if curr_session: + if use_onion_session: + httpd.session_onion = curr_session + elif use_i2p_session: + httpd.session_i2p = curr_session + else: + main_session = curr_session + + send_follow_request(curr_session, + base_dir, nickname, + domain, curr_domain, curr_port, + curr_http_prefix, + following_nickname, + following_domain, + following_actor, + following_port, curr_http_prefix, + False, httpd.federation_list, + httpd.send_threads, + httpd.postLog, + httpd.cached_webfingers, + httpd.person_cache, httpd.debug, + httpd.project_version, + httpd.signing_priv_key_pem, + httpd.domain, + httpd.onion_domain, + httpd.i2p_domain) + + # remove the followed handle from the import list + following_str = following_str.replace(orig_line + '\n', '') + try: + with open(import_filename, 'w+', + encoding='utf-8') as fp_import: + fp_import.write(following_str) + except OSError: + print('EX: unable to remove import 2 ' + line + + ' from ' + import_filename) + return True + return False + + +def run_import_following(base_dir: str, httpd): + """Sends out follow requests for imported following csv files + """ + while True: + time.sleep(20) + + # get a list of accounts on the instance, in random sequence + accounts_list = [] + for _, dirs, _ in os.walk(base_dir + '/accounts'): + for account in dirs: + if '@' not in account: + continue + if not is_account_dir(account): + continue + accounts_list.append(account) + break + if not accounts_list: + continue + + # check if each accounts has an import csv + random.shuffle(accounts_list) + for account in accounts_list: + account_dir = base_dir + '/accounts/' + account + import_filename = account_dir + '/import_following.csv' + + if not os.path.isfile(import_filename): + continue + if not _update_import_following(base_dir, account, httpd, + import_filename): + try: + os.remove(import_filename) + except OSError: + print('EX: unable to remove import file ' + + import_filename) + else: + break + + +def run_import_following_watchdog(project_version: str, httpd) -> None: + """Imports following lists from csv for every account on the instance + """ + print('THREAD: Starting import following watchdog ' + project_version) + import_following_original = \ + httpd.thrImportFollowing.clone(run_import_following) + httpd.thrImportFollowing.start() + while True: + time.sleep(20) + if httpd.thrImportFollowing.is_alive(): + continue + httpd.thrImportFollowing.kill() + print('THREAD: restarting import following watchdog') + httpd.thrImportFollowing = \ + import_following_original.clone(run_import_following) + httpd.thrImportFollowing.start() + print('Restarting import following...') diff --git a/tests.py b/tests.py index 005e26467..78444e709 100644 --- a/tests.py +++ b/tests.py @@ -5408,6 +5408,8 @@ def _test_functions(): 'get_document_loader', 'run_inbox_queue_watchdog', 'run_inbox_queue', + 'run_import_following', + 'run_import_following_watchdog', 'run_post_schedule', 'run_post_schedule_watchdog', 'str2bool', diff --git a/translations/ar.json b/translations/ar.json index e314205fc..ed8db5f38 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "ستتم إضافة تحذيرات المحتوى لما يلي", "nowplaying": "الان العب", "NowPlaying": "الان العب", - "Import and Export": "استيراد وتصدير" + "Import and Export": "استيراد وتصدير", + "Import Follows": "يتبع الاستيراد" } diff --git a/translations/bn.json b/translations/bn.json index 18c83f201..16ebc04e9 100644 --- a/translations/bn.json +++ b/translations/bn.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "নিম্নলিখিত জন্য বিষয়বস্তু সতর্কতা যোগ করা হবে", "nowplaying": "এখন চলছে", "NowPlaying": "এখন চলছে", - "Import and Export": "আমদানি এবং রপ্তানি" + "Import and Export": "আমদানি এবং রপ্তানি", + "Import Follows": "আমদানি অনুসরণ করে" } diff --git a/translations/ca.json b/translations/ca.json index 70fd73081..984363360 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "S'afegiran advertències de contingut per al següent", "nowplaying": "arajugant", "NowPlaying": "AraJugant", - "Import and Export": "Importació i Exportació" + "Import and Export": "Importació i Exportació", + "Import Follows": "Segueix la importació" } diff --git a/translations/cy.json b/translations/cy.json index 691a6352e..fc1506d12 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Bydd rhybuddion cynnwys yn cael eu hychwanegu ar gyfer y canlynol", "nowplaying": "nawrynchwarae", "NowPlaying": "NawrYnChwarae", - "Import and Export": "Mewnforio ac Allforio" + "Import and Export": "Mewnforio ac Allforio", + "Import Follows": "Mewnforio Dilyn" } diff --git a/translations/de.json b/translations/de.json index 0961227c0..3acf03d1b 100644 --- a/translations/de.json +++ b/translations/de.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Inhaltswarnungen werden für Folgendes hinzugefügt", "nowplaying": "läuftgerade", "NowPlaying": "LäuftGerade", - "Import and Export": "Import und Export" + "Import and Export": "Import und Export", + "Import Follows": "Import folgt" } diff --git a/translations/el.json b/translations/el.json index 86895f005..72b231aa8 100644 --- a/translations/el.json +++ b/translations/el.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Θα προστεθούν προειδοποιήσεις περιεχομένου για τα ακόλουθα", "nowplaying": "τώραπαίζει", "NowPlaying": "ΤώραΠαίζει", - "Import and Export": "Εισάγω και εξάγω" + "Import and Export": "Εισάγω και εξάγω", + "Import Follows": "Ακολουθεί εισαγωγή" } diff --git a/translations/en.json b/translations/en.json index da9fd9a5c..9aa14b443 100644 --- a/translations/en.json +++ b/translations/en.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Content warnings will be added for the following", "nowplaying": "nowplaying", "NowPlaying": "NowPlaying", - "Import and Export": "Import and Export" + "Import and Export": "Import and Export", + "Import Follows": "Import Follows" } diff --git a/translations/es.json b/translations/es.json index c8b992c5b..12229db9e 100644 --- a/translations/es.json +++ b/translations/es.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Se agregarán advertencias de contenido para lo siguiente", "nowplaying": "jugandoahora", "NowPlaying": "JugandoAhora", - "Import and Export": "Importar y exportar" + "Import and Export": "Importar y exportar", + "Import Follows": "Importar seguimientos" } diff --git a/translations/fr.json b/translations/fr.json index fb5c7d630..19d2f1e08 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Des avertissements de contenu seront ajoutés pour les éléments suivants", "nowplaying": "lectureencours", "NowPlaying": "LectureEnCours", - "Import and Export": "Importer et exporter" + "Import and Export": "Importer et exporter", + "Import Follows": "Importer suit" } diff --git a/translations/ga.json b/translations/ga.json index e0a9db521..07fd0ac95 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Cuirfear rabhaidh ábhair leis maidir leis na nithe seo a leanas", "nowplaying": "anoisagimirt", "NowPlaying": "AnoisAgImirt", - "Import and Export": "Iompórtáil agus Easpórtáil" + "Import and Export": "Iompórtáil agus Easpórtáil", + "Import Follows": "Leanann Iompórtáil" } diff --git a/translations/hi.json b/translations/hi.json index 04279fab9..d918f0c5c 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "निम्नलिखित के लिए सामग्री चेतावनियां जोड़ दी जाएंगी", "nowplaying": "अब खेल रहे हैं", "NowPlaying": "अब खेल रहे हैं", - "Import and Export": "आयात और निर्यात" + "Import and Export": "आयात और निर्यात", + "Import Follows": "आयात का अनुसरण करता है" } diff --git a/translations/it.json b/translations/it.json index 0e0e63159..5628bde65 100644 --- a/translations/it.json +++ b/translations/it.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Verranno aggiunti avvisi sui contenuti per quanto segue", "nowplaying": "ora giocando", "NowPlaying": "OraGiocando", - "Import and Export": "Importazione e esportazione" + "Import and Export": "Importazione e esportazione", + "Import Follows": "Importa segue" } diff --git a/translations/ja.json b/translations/ja.json index 5560b16eb..841ae9a2e 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "以下のコンテンツ警告が追加されます", "nowplaying": "再生中", "NowPlaying": "再生中", - "Import and Export": "インポートとエクスポート" + "Import and Export": "インポートとエクスポート", + "Import Follows": "インポートフォロー" } diff --git a/translations/ko.json b/translations/ko.json index 1a5616e52..3219cc6b1 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "다음에 대한 콘텐츠 경고가 추가됩니다.", "nowplaying": "지금 재생", "NowPlaying": "지금 재생", - "Import and Export": "가져오기 및 내보내기" + "Import and Export": "가져오기 및 내보내기", + "Import Follows": "가져오기 팔로우" } diff --git a/translations/ku.json b/translations/ku.json index 47b2996ea..3bea33b45 100644 --- a/translations/ku.json +++ b/translations/ku.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Hişyariyên naverokê dê ji bo jêrîn werin zêdekirin", "nowplaying": "nihadilîze", "NowPlaying": "NihaDilîze", - "Import and Export": "Import û Export" + "Import and Export": "Import û Export", + "Import Follows": "Import Follows" } diff --git a/translations/nl.json b/translations/nl.json index a237c02ce..83133657f 100644 --- a/translations/nl.json +++ b/translations/nl.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Er worden inhoudswaarschuwingen toegevoegd voor het volgende:", "nowplaying": "nuaanhetspelen", "NowPlaying": "NuAanHetSpelen", - "Import and Export": "Importeren en exporteren" + "Import and Export": "Importeren en exporteren", + "Import Follows": "Volgt importeren" } diff --git a/translations/oc.json b/translations/oc.json index aade46efe..0f442a0cc 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -566,5 +566,6 @@ "Content warnings will be added for the following": "Content warnings will be added for the following", "nowplaying": "nowplaying", "NowPlaying": "NowPlaying", - "Import and Export": "Import and Export" + "Import and Export": "Import and Export", + "Import Follows": "Import Follows" } diff --git a/translations/pl.json b/translations/pl.json index a101196e7..edb23dfac 100644 --- a/translations/pl.json +++ b/translations/pl.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Ostrzeżenia dotyczące treści zostaną dodane do następujących", "nowplaying": "terazgra", "NowPlaying": "TerazGra", - "Import and Export": "Importuj i eksportuj" + "Import and Export": "Importuj i eksportuj", + "Import Follows": "Importuj obserwuje" } diff --git a/translations/pt.json b/translations/pt.json index 7b3890038..7829ba381 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Avisos de conteúdo serão adicionados para os seguintes", "nowplaying": "agorajogando", "NowPlaying": "AgoraJogando", - "Import and Export": "Importar e exportar" + "Import and Export": "Importar e exportar", + "Import Follows": "Importar seguidores" } diff --git a/translations/ru.json b/translations/ru.json index bd64a7eab..af2e9a37a 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Предупреждения о содержании будут добавлены для следующих", "nowplaying": "сейчасиграет", "NowPlaying": "СейчасИграет", - "Import and Export": "Импорт и экспорт" + "Import and Export": "Импорт и экспорт", + "Import Follows": "Импорт подписок" } diff --git a/translations/sw.json b/translations/sw.json index e24ca61a5..e2848196c 100644 --- a/translations/sw.json +++ b/translations/sw.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Maonyo ya maudhui yataongezwa kwa yafuatayo", "nowplaying": "inachezasasa", "NowPlaying": "InachezaSasa", - "Import and Export": "Ingiza na Hamisha" + "Import and Export": "Ingiza na Hamisha", + "Import Follows": "Ingiza Inafuata" } diff --git a/translations/tr.json b/translations/tr.json index fc195d6b0..a967b9c9c 100644 --- a/translations/tr.json +++ b/translations/tr.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Aşağıdakiler için içerik uyarıları eklenecek", "nowplaying": "şimdioynuyor", "NowPlaying": "ŞimdiOynuyor", - "Import and Export": "İthalat ve ihracat" + "Import and Export": "İthalat ve ihracat", + "Import Follows": "Takipleri İçe Aktar" } diff --git a/translations/uk.json b/translations/uk.json index 202ea9d55..4bf851e7d 100644 --- a/translations/uk.json +++ b/translations/uk.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "Попередження про вміст буде додано для наступних", "nowplaying": "заразграє", "NowPlaying": "ЗаразГрає", - "Import and Export": "Імпорт та експорт" + "Import and Export": "Імпорт та експорт", + "Import Follows": "Імпорт слідує" } diff --git a/translations/yi.json b/translations/yi.json index 33a188989..4145faad2 100644 --- a/translations/yi.json +++ b/translations/yi.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "אינהאַלט וואָרנינגז וועט זיין מוסיף פֿאַר די פאלגענדע", "nowplaying": "איצט פּלייַינג", "NowPlaying": "איצט פּלייַינג", - "Import and Export": "אַרייַנפיר און עקספּאָרט" + "Import and Export": "אַרייַנפיר און עקספּאָרט", + "Import Follows": "אַרייַנפיר גייט" } diff --git a/translations/zh.json b/translations/zh.json index 928ce8e37..702a1019f 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -570,5 +570,6 @@ "Content warnings will be added for the following": "将为以下内容添加内容警告", "nowplaying": "现在玩", "NowPlaying": "现在玩", - "Import and Export": "进出口" + "Import and Export": "进出口", + "Import Follows": "导入关注" } diff --git a/webapp_profile.py b/webapp_profile.py index 70a739a85..585a2741b 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -2003,6 +2003,14 @@ def _html_edit_profile_import_export(nickname: str, domain: str, """Contact Information section of edit profile screen """ edit_profile_form = begin_edit_section(translate['Import and Export']) + + edit_profile_form += \ + '

\n' + edit_profile_form += '