Beginning of federated blocklists

blocking.py

@@ -1815,3 +1815,127 @@ def contains_military_domain(message_str: str) -> bool:
                domain_str + '/' in message_str:
                 return True
     return False
+
+
+def load_federated_blocks_endpoints(base_dir: str) -> []:
+    """Loads endpoint urls for federated blocklists
+    """
+    block_federated_endpoints = []
+    block_api_endpoints_filename = \
+        base_dir + '/accounts/block_api_endpoints.txt'
+    if os.path.isfile(block_api_endpoints_filename):
+        try:
+            with open(block_api_endpoints_filename, 'r',
+                      encoding='utf-8') as fp_ep:
+                block_federated_endpoints = fp_ep.read().split('\n')
+        except OSError:
+            print('EX: unable to load block_api_endpoints.txt')
+    return block_federated_endpoints
+
+
+def update_federated_blocks(session, base_dir: str,
+                            http_prefix: str,
+                            domain: str, domain_full: str,
+                            debug: bool, version: str,
+                            signing_priv_key_pem: str,
+                            max_api_blocks: int) -> []:
+    """Creates block_api.txt
+    """
+    block_federated = []
+
+    if not session:
+        print('WARN: No session for update_federated_blocks')
+        return block_federated
+
+    headers = {
+        'host': domain,
+        'Content-type': 'application/json',
+        'Accept': 'application/json'
+    }
+
+    block_federated_endpoints = load_federated_blocks_endpoints(base_dir)
+
+    new_block_api_str = ''
+    for endpoint in block_federated_endpoints:
+        url = endpoint.strip()
+
+        if debug:
+            print('Block API endpoint: ' + url)
+        blocked_json = get_json(signing_priv_key_pem, session, url, headers,
+                                None, debug, version, http_prefix, None)
+        if not get_json_valid(blocked_json):
+            if debug:
+                print('DEBUG: GET blocked collection failed for c2s to ' + url)
+            continue
+        if isinstance(blocked_json, list):
+            # ensure that the size of the list does not become a form of denial
+            # of service
+            if len(blocked_json) < max_api_blocks:
+                for block_dict in blocked_json:
+                    if not isinstance(block_dict, dict):
+                        continue
+                    if not block_dict.get('username'):
+                        continue
+                    if not isinstance(block_dict['username'], str):
+                        continue
+                    handle = block_dict['username']
+                    if handle.startswith('@'):
+                        handle = handle[1:]
+                    if ' ' in handle or \
+                       ',' in handle or \
+                       ';' in handle or \
+                       '.' not in handle or \
+                       '<' in handle:
+                        continue
+                    new_block_api_str += handle + '\n'
+                    if handle not in block_federated:
+                        block_federated.append(handle)
+
+    block_api_filename = \
+        base_dir + '/accounts/block_api.txt'
+    if not new_block_api_str:
+        if os.path.isfile(block_api_filename):
+            try:
+                os.remove(block_api_filename)
+            except OSError:
+                print('EX: unable to remove block api: ' + block_api_filename)
+    else:
+        try:
+            with open(block_api_filename, 'w+', encoding='utf-8') as fp_api:
+                fp_api.write(new_block_api_str)
+        except OSError:
+            print('EX: unable to write block_api.txt')
+
+    return block_federated
+
+
+def save_block_federated_endpoints(base_dir: str,
+                                   block_federated_endpoints: []) -> []:
+    """Saves a list of blocking API endpoints
+    """
+    block_api_endpoints_filename = \
+        base_dir + '/accounts/block_api_endpoints.txt'
+    result = []
+    block_federated_endpoints_str = ''
+    for endpoint in block_federated_endpoints:
+        if not endpoint:
+            continue
+        if '.' not in endpoint or \
+           ' ' in endpoint or \
+           '<' in endpoint or \
+           ',' in endpoint or \
+           ';' in endpoint:
+            continue
+        if endpoint.startswith('@'):
+            endpoint = endpoint[1:]
+        if not endpoint:
+            continue
+        block_federated_endpoints_str += endpoint.strip() + '\n'
+        result.append(endpoint)
+    try:
+        with open(block_api_endpoints_filename, 'w+',
+                  encoding='utf-8') as fp_api:
+            fp_api.write(block_federated_endpoints_str)
+    except OSError:
+        print('EX: unable to write block_api_endpoints.txt')
+    return result

daemon.py

@@ -160,6 +160,9 @@ from media import path_is_transcript
 from media import path_is_audio
 from cwlists import get_cw_list_variable
 from cwlists import load_cw_lists
+from blocking import save_block_federated_endpoints
+from blocking import load_federated_blocks_endpoints
+from blocking import update_federated_blocks
 from blocking import contains_military_domain
 from blocking import load_blocked_military
 from blocking import save_blocked_military
@@ -8456,6 +8459,20 @@ class PubServer(BaseHTTPRequestHandler):
                                         print('EX: unable to delete ' +
                                               buy_sites_filename)
 
+                        # save blocking API endpoints
+                        block_ep_new = []
+                        if fields.get('blockFederated'):
+                            block_federated_str = \
+                                fields['blockFederated']
+                            block_ep_new = \
+                                block_federated_str.split('\n')
+                        if str(self.server.block_federated_endpoints) != \
+                           str(block_ep_new):
+                            base_dir = self.server.base_dir
+                            self.server.block_federated_endpoints = \
+                                save_block_federated_endpoints(base_dir,
+                                                               block_ep_new)
+
                         # save peertube instances list
                         peertube_instances_file = \
                             base_dir + '/accounts/peertube.txt'
@@ -17311,7 +17328,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.max_recent_posts,
                                     self.server.reverse_sequence,
                                     self.server.buy_sites,
-                                    self.server.block_military)
+                                    self.server.block_military,
+                                    self.server.block_federated_endpoints)
             if msg:
                 msg = msg.encode('utf-8')
                 msglen = len(msg)
@@ -25259,6 +25277,22 @@ def run_daemon(no_of_books: int,
     # this is the instance actor private key
     httpd.signing_priv_key_pem = get_instance_actor_key(base_dir, domain)
 
+    # load federated blocklists
+    httpd.max_api_blocks = 32000
+    httpd.block_federated_endpoints = \
+        load_federated_blocks_endpoints(base_dir)
+    httpd.block_federated = []
+    if not unit_test:
+        curr_session = create_session(proxy_type)
+        if curr_session:
+            httpd.block_federated = \
+                update_federated_blocks(curr_session, base_dir,
+                                        http_prefix,
+                                        domain, httpd.domain_full,
+                                        debug, project_version,
+                                        httpd.signing_priv_key_pem,
+                                        httpd.max_api_blocks)
+
     # threads used for checking for actor changes when clicking on
     # avatar icon / person options
     httpd.thrCheckActor = {}

translations/ar.json

@@ -656,5 +656,6 @@
     "reading": "قراءة",
     "Birthday": "عيد ميلاد",
     "Only show reminders": "عرض التذكيرات فقط",
-    "Show all events": "عرض كافة الأحداث"
+    "Show all events": "عرض كافة الأحداث",
+    "Blocking API endpoints": "حظر نقاط نهاية API"
 }

translations/bn.json

@@ -656,5 +656,6 @@
     "reading": "পড়া",
     "Birthday": "জন্মদিন",
     "Only show reminders": "শুধুমাত্র অনুস্মারক দেখান",
-    "Show all events": "সমস্ত ঘটনা দেখান"
+    "Show all events": "সমস্ত ঘটনা দেখান",
+    "Blocking API endpoints": "এপিআই এন্ডপয়েন্ট ব্লক করা"
 }

translations/ca.json

@@ -656,5 +656,6 @@
     "reading": "lectura",
     "Birthday": "Aniversari",
     "Only show reminders": "Mostra només recordatoris",
-    "Show all events": "Mostra tots els esdeveniments"
+    "Show all events": "Mostra tots els esdeveniments",
+    "Blocking API endpoints": "Bloqueig de punts finals de l'API"
 }

translations/cy.json

@@ -656,5 +656,6 @@
     "reading": "darllen",
     "Birthday": "Penblwydd",
     "Only show reminders": "Dangoswch nodiadau atgoffa yn unig",
-    "Show all events": "Dangos pob digwyddiad"
+    "Show all events": "Dangos pob digwyddiad",
+    "Blocking API endpoints": "Rhwystro pwyntiau terfyn API"
 }

translations/de.json

@@ -656,5 +656,6 @@
     "reading": "lektüre",
     "Birthday": "Geburtstag",
     "Only show reminders": "Nur Erinnerungen anzeigen",
-    "Show all events": "Alle Veranstaltungen anzeigen"
+    "Show all events": "Alle Veranstaltungen anzeigen",
+    "Blocking API endpoints": "Blockieren von API-Endpunkten"
 }

translations/el.json

@@ -656,5 +656,6 @@
     "reading": "ΑΝΑΓΝΩΣΗ",
     "Birthday": "Γενέθλια",
     "Only show reminders": "Εμφάνιση μόνο υπενθυμίσεων",
-    "Show all events": "Εμφάνιση όλων των συμβάντων"
+    "Show all events": "Εμφάνιση όλων των συμβάντων",
+    "Blocking API endpoints": "Αποκλεισμός τελικών σημείων API"
 }

translations/en.json

@@ -656,5 +656,6 @@
     "reading": "reading",
     "Birthday": "Birthday",
     "Only show reminders": "Only show reminders",
-    "Show all events": "Show all events"
+    "Show all events": "Show all events",
+    "Blocking API endpoints": "Blocking API endpoints"
 }

translations/es.json

@@ -656,5 +656,6 @@
     "reading": "lectura",
     "Birthday": "Cumpleaños",
     "Only show reminders": "Mostrar solo recordatorios",
-    "Show all events": "Mostrar todos los eventos"
+    "Show all events": "Mostrar todos los eventos",
+    "Blocking API endpoints": "Bloqueo de puntos finales de API"
 }

translations/fa.json

@@ -656,5 +656,6 @@
     "reading": "خواندن",
     "Birthday": "روز تولد",
     "Only show reminders": "فقط یادآوری ها را نشان دهید",
-    "Show all events": "نمایش همه رویدادها"
+    "Show all events": "نمایش همه رویدادها",
+    "Blocking API endpoints": "مسدود کردن نقاط پایانی API"
 }

translations/fr.json

@@ -656,5 +656,6 @@
     "reading": "en lisant",
     "Birthday": "Anniversaire",
     "Only show reminders": "Afficher uniquement les rappels",
-    "Show all events": "Afficher tous les événements"
+    "Show all events": "Afficher tous les événements",
+    "Blocking API endpoints": "Blocage des points de terminaison de l'API"
 }

translations/ga.json

@@ -656,5 +656,6 @@
     "reading": "ag léamh",
     "Birthday": "Breithlá",
     "Only show reminders": "Ná taispeáin ach meabhrúcháin",
-    "Show all events": "Taispeáin gach imeacht"
+    "Show all events": "Taispeáin gach imeacht",
+    "Blocking API endpoints": "Ag cur bac ar chríochphointí API"
 }

translations/he.json

@@ -656,5 +656,6 @@
     "reading": "קריאה",
     "Birthday": "יום הולדת",
     "Only show reminders": "הצג רק תזכורות",
-    "Show all events": "הצג את כל האירועים"
+    "Show all events": "הצג את כל האירועים",
+    "Blocking API endpoints": "חסימת נקודות קצה של API"
 }

translations/hi.json

@@ -656,5 +656,6 @@
     "reading": "पढ़ना",
     "Birthday": "जन्मदिन",
     "Only show reminders": "केवल अनुस्मारक दिखाएँ",
-    "Show all events": "सभी घटनाएँ दिखाएँ"
+    "Show all events": "सभी घटनाएँ दिखाएँ",
+    "Blocking API endpoints": "एपीआई समापनबिंदुओं को अवरुद्ध करना"
 }

translations/it.json

@@ -656,5 +656,6 @@
     "reading": "lettura",
     "Birthday": "Compleanno",
     "Only show reminders": "Mostra solo promemoria",
-    "Show all events": "Mostra tutti gli eventi"
+    "Show all events": "Mostra tutti gli eventi",
+    "Blocking API endpoints": "Blocco degli endpoint API"
 }

translations/ja.json

@@ -656,5 +656,6 @@
     "reading": "読む",
     "Birthday": "誕生日",
     "Only show reminders": "リマインダーのみを表示",
-    "Show all events": "すべてのイベントを表示"
+    "Show all events": "すべてのイベントを表示",
+    "Blocking API endpoints": "APIエンドポイントのブロック"
 }

translations/ko.json

@@ -656,5 +656,6 @@
     "reading": "독서",
     "Birthday": "생일",
     "Only show reminders": "알림만 표시",
-    "Show all events": "모든 이벤트 표시"
+    "Show all events": "모든 이벤트 표시",
+    "Blocking API endpoints": "API 엔드포인트 차단"
 }

translations/ku.json

@@ -656,5 +656,6 @@
     "reading": "xwendinî",
     "Birthday": "Rojbûn",
     "Only show reminders": "Tenê bîranînan nîşan bidin",
-    "Show all events": "Hemî bûyeran nîşan bide"
+    "Show all events": "Hemî bûyeran nîşan bide",
+    "Blocking API endpoints": "Astengkirina xalên dawiya API-ê"
 }

translations/nl.json

@@ -656,5 +656,6 @@
     "reading": "lezing",
     "Birthday": "Verjaardag",
     "Only show reminders": "Toon alleen herinneringen",
-    "Show all events": "Toon alle evenementen"
+    "Show all events": "Toon alle evenementen",
+    "Blocking API endpoints": "API-eindpunten blokkeren"
 }

translations/oc.json

@@ -652,5 +652,6 @@
     "reading": "reading",
     "Birthday": "Birthday",
     "Only show reminders": "Only show reminders",
-    "Show all events": "Show all events"
+    "Show all events": "Show all events",
+    "Blocking API endpoints": "Blocking API endpoints"
 }

translations/pl.json

@@ -656,5 +656,6 @@
     "reading": "czytanie",
     "Birthday": "Urodziny",
     "Only show reminders": "Pokazuj tylko przypomnienia",
-    "Show all events": "Pokaż wszystkie wydarzenia"
+    "Show all events": "Pokaż wszystkie wydarzenia",
+    "Blocking API endpoints": "Blokowanie punktów końcowych API"
 }

translations/pt.json

@@ -656,5 +656,6 @@
     "reading": "leitura",
     "Birthday": "Aniversário",
     "Only show reminders": "Mostrar apenas lembretes",
-    "Show all events": "Mostrar todos os eventos"
+    "Show all events": "Mostrar todos os eventos",
+    "Blocking API endpoints": "Bloqueio de endpoints de API"
 }

translations/ru.json

@@ -656,5 +656,6 @@
     "reading": "чтение",
     "Birthday": "День рождения",
     "Only show reminders": "Показывать только напоминания",
-    "Show all events": "Показать все события"
+    "Show all events": "Показать все события",
+    "Blocking API endpoints": "Блокировка конечных точек API"
 }

translations/sw.json

@@ -656,5 +656,6 @@
     "reading": "kusoma",
     "Birthday": "Siku ya kuzaliwa",
     "Only show reminders": "Onyesha vikumbusho pekee",
-    "Show all events": "Onyesha matukio yote"
+    "Show all events": "Onyesha matukio yote",
+    "Blocking API endpoints": "Inazuia miisho ya API"
 }

translations/tr.json

@@ -656,5 +656,6 @@
     "reading": "okuma",
     "Birthday": "Doğum günü",
     "Only show reminders": "Yalnızca hatırlatıcıları göster",
-    "Show all events": "Tüm etkinlikleri göster"
+    "Show all events": "Tüm etkinlikleri göster",
+    "Blocking API endpoints": "API uç noktalarını engelleme"
 }

translations/uk.json

@@ -656,5 +656,6 @@
     "reading": "читання",
     "Birthday": "день народження",
     "Only show reminders": "Показувати лише нагадування",
-    "Show all events": "Показати всі події"
+    "Show all events": "Показати всі події",
+    "Blocking API endpoints": "Блокування кінцевих точок API"
 }

translations/yi.json

@@ -656,5 +656,6 @@
     "reading": "לייענען",
     "Birthday": "דיין געבורסטאָג",
     "Only show reminders": "בלויז ווייַזן רימיינדערז",
-    "Show all events": "ווייַזן אַלע געשעענישן"
+    "Show all events": "ווייַזן אַלע געשעענישן",
+    "Blocking API endpoints": "בלאַקינג אַפּי ענדפּאָינץ"
 }

translations/zh.json

@@ -656,5 +656,6 @@
     "reading": "阅读",
     "Birthday": "生日",
     "Only show reminders": "只显示提醒",
-    "Show all events": "显示所有活动"
+    "Show all events": "显示所有活动",
+    "Blocking API endpoints": "阻止 API 端点"
 }

webapp_profile.py

@@ -2223,7 +2223,8 @@ def _html_edit_profile_filtering(base_dir: str, nickname: str, domain: str,
                                  crawlers_allowed: str,
                                  translate: {}, reply_interval_hours: int,
                                  cw_lists: {}, lists_enabled: str,
-                                 buy_sites: {}, block_military: {}) -> str:
+                                 buy_sites: {}, block_military: {},
+                                 block_federated_endpoints: []) -> str:
     """Filtering and blocking section of edit profile screen
     """
     filter_str = ''
@@ -2447,6 +2448,16 @@ def _html_edit_profile_filtering(base_dir: str, nickname: str, domain: str,
         edit_profile_form += \
             edit_check_box(idx, 'blockMilitary', block_mil)
 
+        block_federated_endpoints_list_str = ''
+        for block_api_url in block_federated_endpoints:
+            block_federated_endpoints_list_str += block_api_url.strip() + '\n'
+        block_federated_str = "Blocking API endpoints"
+        edit_profile_form += \
+            edit_text_area(translate[block_federated_str], None,
+                           'blockFederated',
+                           block_federated_endpoints_list_str,
+                           200, '', False)
+
         cw_lists_str = ''
         for name, _ in cw_lists.items():
             variablename = get_cw_list_variable(name)
@@ -2943,7 +2954,8 @@ def html_edit_profile(server, translate: {},
                       max_recent_posts: int,
                       reverse_sequence: [],
                       buy_sites: {},
-                      block_military: {}) -> str:
+                      block_military: {},
+                      block_federated_endpoints: []) -> str:
     """Shows the edit profile screen
     """
     path = path.replace('/inbox', '').replace('/outbox', '')
@@ -3214,7 +3226,7 @@ def html_edit_profile(server, translate: {},
                                      user_agents_blocked, crawlers_allowed,
                                      translate, reply_interval_hours,
                                      cw_lists, lists_enabled, buy_sites,
-                                     block_military)
+                                     block_military, block_federated_endpoints)
 
     # git projects section
     edit_profile_form += \