__filename__ = "daemon.py" __author__ = "Bob Mottram" __license__ = "AGPL3+" __version__ = "1.5.0" __maintainer__ = "Bob Mottram" __email__ = "bob@libreserver.org" __status__ = "Production" __module_group__ = "Core" from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer, HTTPServer import sys import time import datetime import os from socket import error as SocketError import errno from functools import partial # for saving images from hashlib import md5 from metadata import metadata_custom_emoji from person import update_memorial_flags from person import clear_person_qrcodes from person import create_shared_inbox from person import create_news_inbox from keys import get_instance_actor_key from posts import expire_cache from inbox import run_inbox_queue from inbox import run_inbox_queue_watchdog from follow import create_initial_last_seen from threads import begin_thread from threads import thread_with_trace from threads import remove_dormant_threads from media import path_is_video from media import path_is_audio from cwlists import load_cw_lists from blocking import run_federated_blocks_daemon from blocking import load_federated_blocks_endpoints from blocking import load_blocked_military from blocking import update_blocked_cache from blocking import set_broch_mode from blocking import get_domain_blocklist from webapp_utils import load_buy_sites from webapp_accesskeys import load_access_keys_for_accounts from webapp_media import load_peertube_instances from shares import run_federated_shares_daemon from shares import run_federated_shares_watchdog from shares import create_shared_item_federation_token from shares import generate_shared_item_federation_tokens from shares import expire_shares from categories import load_city_hashtags from categories import update_hashtag_categories from languages import load_default_post_languages from utils import string_contains from utils import check_bad_path from utils import acct_handle_dir from utils import load_reverse_timeline from utils import load_min_images_for_accounts from utils import load_account_timezones from utils import load_translations_from_file from utils import load_bold_reading from utils import load_hide_follows from utils import decoded_host from utils import get_full_domain from utils import media_file_mime_type from utils import set_config_param from utils import get_config_param from utils import load_json from utils import is_image_file from content import load_auto_cw_cache from content import load_dogwhistles from theme import scan_themes_for_scripts from theme import is_news_theme_name from theme import get_text_mode_banner from theme import set_news_avatar from schedule import run_post_schedule from schedule import run_post_schedule_watchdog from happening import dav_propfind_response from happening import dav_put_response from happening import dav_report_response from happening import dav_delete_response from newswire import load_hashtag_categories from newsdaemon import run_newswire_watchdog from newsdaemon import run_newswire_daemon from fitnessFunctions import fitness_thread from siteactive import load_unavailable_sites from crawlers import load_known_web_bots from qrcode import save_domain_qrcode from importFollowing import run_import_following_watchdog from relationships import update_moved_actors from daemon_get import daemon_http_get from daemon_post import daemon_http_post from httpcodes import http_200 from httpcodes import http_201 from httpcodes import http_207 from httpcodes import http_403 from httpcodes import http_404 from httpcodes import http_304 from httpcodes import http_400 from httpcodes import write2 from httpheaders import set_headers from httpheaders import set_headers_head from daemon_utils import has_accept from daemon_utils import is_authorized class PubServer(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' def handle_error(self, request, client_address): """HTTP server error handling """ print('ERROR: http server error: ' + str(request) + ', ' + str(client_address)) def do_GET(self): daemon_http_get(self) def _dav_handler(self, endpoint_type: str, debug: bool): calling_domain = self.server.domain_full if not has_accept(self, calling_domain): http_400(self) return accept_str = self.headers['Accept'] if 'application/xml' not in accept_str: if debug: print(endpoint_type.upper() + ' is not of xml type') http_400(self) return if not self.headers.get('Content-length'): print(endpoint_type.upper() + ' has no content-length') http_400(self) return # check that the content length string is not too long if isinstance(self.headers['Content-length'], str): max_content_size = len(str(self.server.maxMessageLength)) if len(self.headers['Content-length']) > max_content_size: http_400(self) return length = int(self.headers['Content-length']) if length > self.server.max_post_length: print(endpoint_type.upper() + ' request size too large ' + self.path) http_400(self) return if not self.path.startswith('/calendars/'): print(endpoint_type.upper() + ' without /calendars ' + self.path) http_404(self, 145) return if debug: print(endpoint_type.upper() + ' checking authorization') if not is_authorized(self): print(endpoint_type.upper() + ' not authorized') http_403(self) return nickname = self.path.split('/calendars/')[1] if '/' in nickname: nickname = nickname.split('/')[0] if not nickname: print(endpoint_type.upper() + ' no nickname ' + self.path) http_400(self) return if not os.path.isdir(self.server.base_dir + '/accounts/' + nickname + '@' + self.server.domain): print(endpoint_type.upper() + ' for non-existent account ' + self.path) http_404(self, 146) return propfind_bytes = None try: propfind_bytes = self.rfile.read(length) except SocketError as ex: if ex.errno == errno.ECONNRESET: print('EX: ' + endpoint_type.upper() + ' connection reset by peer') else: print('EX: ' + endpoint_type.upper() + ' socket error') http_400(self) return except ValueError as ex: print('EX: ' + endpoint_type.upper() + ' rfile.read failed, ' + str(ex)) http_400(self) return if not propfind_bytes: http_404(self, 147) return propfind_xml = propfind_bytes.decode('utf-8') response_str = None if endpoint_type == 'propfind': response_str = \ dav_propfind_response(nickname, propfind_xml) elif endpoint_type == 'put': response_str = \ dav_put_response(self.server.base_dir, nickname, self.server.domain, propfind_xml, self.server.http_prefix, self.server.system_language, self.server.recent_dav_etags) elif endpoint_type == 'report': curr_etag = None if self.headers.get('ETag'): curr_etag = self.headers['ETag'] elif self.headers.get('Etag'): curr_etag = self.headers['Etag'] response_str = \ dav_report_response(self.server.base_dir, nickname, self.server.domain, propfind_xml, self.server.person_cache, self.server.http_prefix, curr_etag, self.server.recent_dav_etags, self.server.domain_full, self.server.system_language) elif endpoint_type == 'delete': response_str = \ dav_delete_response(self.server.base_dir, nickname, self.server.domain, self.path, self.server.http_prefix, debug, self.server.recent_posts_cache) if not response_str: http_404(self, 148) return if response_str == 'Not modified': if endpoint_type == 'put': http_200(self) return http_304(self) return if response_str.startswith('ETag:') and endpoint_type == 'put': response_etag = response_str.split('ETag:', 1)[1] http_201(self, response_etag) elif response_str != 'Ok': message_xml = response_str.encode('utf-8') message_xml_len = len(message_xml) set_headers(self, 'application/xml; charset=utf-8', message_xml_len, None, calling_domain, False) write2(self, message_xml) if 'multistatus' in response_str: return http_207(self) http_200(self) def do_PROPFIND(self): if self.server.starting_daemon: return if check_bad_path(self.path): http_400(self) return self._dav_handler('propfind', self.server.debug) def do_PUT(self): if self.server.starting_daemon: return if check_bad_path(self.path): http_400(self) return self._dav_handler('put', self.server.debug) def do_REPORT(self): if self.server.starting_daemon: return if check_bad_path(self.path): http_400(self) return self._dav_handler('report', self.server.debug) def do_DELETE(self): if self.server.starting_daemon: return if check_bad_path(self.path): http_400(self) return self._dav_handler('delete', self.server.debug) def do_HEAD(self): if self.server.starting_daemon: return if check_bad_path(self.path): http_400(self) return calling_domain = self.server.domain_full if self.headers.get('Host'): calling_domain = decoded_host(self.headers['Host']) if self.server.onion_domain: if calling_domain not in (self.server.domain, self.server.domain_full, self.server.onion_domain): print('HEAD domain blocked: ' + calling_domain) http_400(self) return else: if calling_domain not in (self.server.domain, self.server.domain_full): print('HEAD domain blocked: ' + calling_domain) http_400(self) return check_path = self.path etag = None file_length = -1 last_modified_time_str = None if string_contains(self.path, ('/media/', '/accounts/avatars/', '/accounts/headers/')): if is_image_file(self.path) or \ path_is_video(self.path) or \ path_is_audio(self.path): if '/media/' in self.path: media_str = self.path.split('/media/')[1] media_filename = \ self.server.base_dir + '/media/' + media_str elif '/accounts/avatars/' in self.path: avatar_file = self.path.split('/accounts/avatars/')[1] if '/' not in avatar_file: http_404(self, 149) return nickname = avatar_file.split('/')[0] avatar_file = avatar_file.split('/')[1] avatar_file_ext = avatar_file.split('.')[-1] # remove any numbers, eg. avatar123.png becomes avatar.png if avatar_file.startswith('avatar'): avatar_file = 'avatar.' + avatar_file_ext media_filename = \ self.server.base_dir + '/accounts/' + \ nickname + '@' + self.server.domain + '/' + \ avatar_file else: banner_file = self.path.split('/accounts/headers/')[1] if '/' not in banner_file: http_404(self, 150) return nickname = banner_file.split('/')[0] banner_file = banner_file.split('/')[1] banner_file_ext = banner_file.split('.')[-1] # remove any numbers, eg. banner123.png becomes banner.png if banner_file.startswith('banner'): banner_file = 'banner.' + banner_file_ext media_filename = \ self.server.base_dir + '/accounts/' + \ nickname + '@' + self.server.domain + '/' + \ banner_file if os.path.isfile(media_filename): check_path = media_filename file_length = os.path.getsize(media_filename) media_tm = os.path.getmtime(media_filename) last_modified_time = \ datetime.datetime.fromtimestamp(media_tm, datetime.timezone.utc) time_format_str = '%a, %d %b %Y %H:%M:%S GMT' last_modified_time_str = \ last_modified_time.strftime(time_format_str) media_tag_filename = media_filename + '.etag' if os.path.isfile(media_tag_filename): try: with open(media_tag_filename, 'r', encoding='utf-8') as efile: etag = efile.read() except OSError: print('EX: do_HEAD unable to read ' + media_tag_filename) else: media_binary = None try: with open(media_filename, 'rb') as av_file: media_binary = av_file.read() except OSError: print('EX: unable to read media binary ' + media_filename) if media_binary: etag = md5(media_binary).hexdigest() # nosec try: with open(media_tag_filename, 'w+', encoding='utf-8') as efile: efile.write(etag) except OSError: print('EX: do_HEAD unable to write ' + media_tag_filename) else: http_404(self, 151) return media_file_type = media_file_mime_type(check_path) set_headers_head(self, media_file_type, file_length, etag, calling_domain, False, last_modified_time_str) def do_POST(self): daemon_http_post(self) class PubServerUnitTest(PubServer): protocol_version = 'HTTP/1.0' class EpicyonServer(ThreadingHTTPServer): def handle_error(self, request, client_address): # surpress connection reset errors cls, e_ret = sys.exc_info()[:2] if cls is ConnectionResetError: if e_ret.errno != errno.ECONNRESET: print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e_ret)) elif cls is BrokenPipeError: pass else: print('ERROR: (EpicyonServer) ' + str(cls) + ", " + str(e_ret)) return HTTPServer.handle_error(self, request, client_address) def run_posts_queue(base_dir: str, send_threads: [], debug: bool, timeout_mins: int) -> None: """Manages the threads used to send posts """ while True: time.sleep(1) remove_dormant_threads(base_dir, send_threads, debug, timeout_mins) def run_shares_expire(version_number: str, base_dir: str, httpd) -> None: """Expires shares as needed """ while True: time.sleep(120) expire_shares(base_dir, httpd.max_shares_on_profile, httpd.person_cache) def run_posts_watchdog(project_version: str, httpd) -> None: """This tries to keep the posts thread running even if it dies """ print('THREAD: Starting posts queue watchdog') posts_queue_original = httpd.thrPostsQueue.clone(run_posts_queue) begin_thread(httpd.thrPostsQueue, 'run_posts_watchdog') while True: time.sleep(20) if httpd.thrPostsQueue.is_alive(): continue httpd.thrPostsQueue.kill() print('THREAD: restarting posts queue') httpd.thrPostsQueue = posts_queue_original.clone(run_posts_queue) begin_thread(httpd.thrPostsQueue, 'run_posts_watchdog 2') print('Restarting posts queue...') def run_shares_expire_watchdog(project_version: str, httpd) -> None: """This tries to keep the shares expiry thread running even if it dies """ print('THREAD: Starting shares expiry watchdog') shares_expire_original = httpd.thrSharesExpire.clone(run_shares_expire) begin_thread(httpd.thrSharesExpire, 'run_shares_expire_watchdog') while True: time.sleep(20) if httpd.thrSharesExpire.is_alive(): continue httpd.thrSharesExpire.kill() print('THREAD: restarting shares watchdog') httpd.thrSharesExpire = shares_expire_original.clone(run_shares_expire) begin_thread(httpd.thrSharesExpire, 'run_shares_expire_watchdog 2') print('Restarting shares expiry...') def load_tokens(base_dir: str, tokens_dict: {}, tokens_lookup: {}) -> None: """Loads shared items access tokens for each account """ for _, dirs, _ in os.walk(base_dir + '/accounts'): for handle in dirs: if '@' in handle: token_filename = acct_handle_dir(base_dir, handle) + '/.token' if not os.path.isfile(token_filename): continue nickname = handle.split('@')[0] token = None try: with open(token_filename, 'r', encoding='utf-8') as fp_tok: token = fp_tok.read() except OSError as ex: print('WARN: Unable to read token for ' + nickname + ' ' + str(ex)) if not token: continue tokens_dict[nickname] = token tokens_lookup[token] = nickname break def run_daemon(no_of_books: int, public_replies_unlisted: int, max_shares_on_profile: int, max_hashtags: int, map_format: str, clacks: str, preferred_podcast_formats: [], check_actor_timeout: int, crawlers_allowed: [], dyslexic_font: bool, content_license_url: str, lists_enabled: str, default_reply_interval_hrs: int, low_bandwidth: bool, max_like_count: int, shared_items_federated_domains: [], user_agents_blocked: [], log_login_failures: bool, city: str, show_node_info_accounts: bool, show_node_info_version: bool, broch_mode: bool, verify_all_signatures: bool, send_threads_timeout_mins: int, dormant_months: int, max_newswire_posts: int, allow_local_network_access: bool, max_feed_item_size_kb: int, publish_button_at_top: bool, rss_icon_at_top: bool, icons_as_buttons: bool, full_width_tl_button_header: bool, show_publish_as_icon: bool, max_followers: int, max_news_posts: int, max_mirrored_articles: int, max_newswire_feed_size_kb: int, max_newswire_posts_per_source: int, show_published_date_only: bool, voting_time_mins: int, positive_voting: bool, newswire_votes_threshold: int, news_instance: bool, blogs_instance: bool, media_instance: bool, max_recent_posts: int, enable_shared_inbox: bool, registration: bool, language: str, project_version: str, instance_id: str, client_to_server: bool, base_dir: str, domain: str, onion_domain: str, i2p_domain: str, yt_replace_domain: str, twitter_replacement_domain: str, port: int = 80, proxy_port: int = 80, http_prefix: str = 'https', fed_list: [] = [], max_mentions: int = 10, max_emoji: int = 10, secure_mode: bool = False, proxy_type: str = None, max_replies: int = 64, domain_max_posts_per_day: int = 8640, account_max_posts_per_day: int = 864, allow_deletion: bool = False, debug: bool = False, unit_test: bool = False, instance_only_skills_search: bool = False, send_threads: [] = [], manual_follower_approval: bool = True) -> None: if len(domain) == 0: domain = 'localhost' if '.' not in domain: if domain != 'localhost': print('Invalid domain: ' + domain) return update_moved_actors(base_dir, debug) if unit_test: server_address = (domain, proxy_port) pub_handler = partial(PubServerUnitTest) else: server_address = ('', proxy_port) pub_handler = partial(PubServer) if not os.path.isdir(base_dir + '/accounts'): print('Creating accounts directory') os.mkdir(base_dir + '/accounts') try: httpd = EpicyonServer(server_address, pub_handler) except SocketError as ex: if ex.errno == errno.ECONNREFUSED: print('EX: HTTP server address is already in use. ' + str(server_address)) return False print('EX: HTTP server failed to start. ' + str(ex)) print('server_address: ' + str(server_address)) return False httpd.starting_daemon = True # number of book events which show on profile screens httpd.no_of_books = no_of_books # initialise federated blocklists httpd.max_api_blocks = 32000 httpd.block_federated_endpoints = \ load_federated_blocks_endpoints(base_dir) httpd.block_federated = [] # cache storing recent book events httpd.books_cache = {} httpd.max_recent_books = 1000 httpd.max_cached_readers = 24 # cache for automatic content warnings httpd.auto_cw_cache = load_auto_cw_cache(base_dir) # list of websites which are currently down httpd.sites_unavailable = load_unavailable_sites(base_dir) # maximum number of shared items attached to actors, as in # https://codeberg.org/fediverse/fep/src/branch/main/fep/0837/fep-0837.md httpd.max_shares_on_profile = max_shares_on_profile # load a list of nicknames for accounts blocking military instances httpd.block_military = load_blocked_military(base_dir) # scan the theme directory for any svg files containing scripts assert not scan_themes_for_scripts(base_dir) # lock for followers synchronization httpd.followers_synchronization = False # cache containing followers synchronization hashes and json httpd.followers_sync_cache = {} # permitted sites from which the buy button may be displayed httpd.buy_sites = load_buy_sites(base_dir) # which accounts should minimize all attached images by default httpd.min_images_for_accounts = load_min_images_for_accounts(base_dir) # default language for each account when creating a new post httpd.default_post_language = load_default_post_languages(base_dir) # caches css files httpd.css_cache = {} httpd.reverse_sequence = load_reverse_timeline(base_dir) httpd.clacks = get_config_param(base_dir, 'clacks') if not httpd.clacks: if clacks: httpd.clacks = clacks else: httpd.clacks = 'GNU Natalie Nguyen' httpd.public_replies_unlisted = public_replies_unlisted # load a list of dogwhistle words dogwhistles_filename = base_dir + '/accounts/dogwhistles.txt' if not os.path.isfile(dogwhistles_filename): dogwhistles_filename = base_dir + '/default_dogwhistles.txt' httpd.dogwhistles = load_dogwhistles(dogwhistles_filename) # list of preferred podcast formats # eg ['audio/opus', 'audio/mp3', 'audio/speex'] httpd.preferred_podcast_formats = preferred_podcast_formats # for each account, whether bold reading is enabled httpd.bold_reading = load_bold_reading(base_dir) # whether to hide follows on profile screen for each account httpd.hide_follows = load_hide_follows(base_dir) httpd.account_timezone = load_account_timezones(base_dir) httpd.post_to_nickname = None httpd.nodeinfo_is_active = False httpd.security_txt_is_active = False httpd.vcard_is_active = False httpd.masto_api_is_active = False # use kml or gpx format for hashtag maps httpd.map_format = map_format.lower() httpd.dyslexic_font = dyslexic_font # license for content of the instance if not content_license_url: content_license_url = 'https://creativecommons.org/licenses/by-nc/4.0' httpd.content_license_url = content_license_url httpd.dm_license_url = '' # fitness metrics fitness_filename = base_dir + '/accounts/fitness.json' httpd.fitness = {} if os.path.isfile(fitness_filename): fitness = load_json(fitness_filename) if fitness is not None: httpd.fitness = fitness # initialize authorized fetch key httpd.signing_priv_key_pem = None httpd.show_node_info_accounts = show_node_info_accounts httpd.show_node_info_version = show_node_info_version # ASCII/ANSI text banner used in shell browsers, such as Lynx httpd.text_mode_banner = get_text_mode_banner(base_dir) # key shortcuts SHIFT + ALT + [key] httpd.access_keys = { 'Page up': ',', 'Page down': '.', 'submitButton': 'y', 'followButton': 'f', 'moveButton': 'm', 'blockButton': 'b', 'infoButton': 'i', 'snoozeButton': 's', 'reportButton': '[', 'viewButton': 'v', 'unblockButton': 'u', 'enterPetname': 'p', 'enterNotes': 'n', 'menuTimeline': 't', 'menuEdit': 'e', 'menuThemeDesigner': 'z', 'menuProfile': 'p', 'menuInbox': 'i', 'menuSearch': '/', 'menuNewPost': 'n', 'menuNewBlog': '0', 'menuCalendar': 'c', 'menuDM': 'd', 'menuReplies': 'r', 'menuOutbox': 's', 'menuBookmarks': 'q', 'menuShares': 'h', 'menuWanted': 'w', 'menuReadingStatus': '=', 'menuBlogs': 'b', 'menuNewswire': '#', 'menuLinks': 'l', 'menuMedia': 'm', 'menuModeration': 'o', 'menuFollowing': 'f', 'menuFollowers': 'g', 'menuRoles': 'o', 'menuSkills': 'a', 'menuLogout': 'x', 'menuKeys': 'k', 'Public': 'p', 'Reminder': 'r' } # timeout used when getting rss feeds httpd.rss_timeout_sec = 20 # timeout used when checking for actor changes when clicking an avatar # and entering person options screen if check_actor_timeout < 2: check_actor_timeout = 2 httpd.check_actor_timeout = check_actor_timeout # how many hours after a post was published can a reply be made default_reply_interval_hrs = 9999999 httpd.default_reply_interval_hrs = default_reply_interval_hrs # recent caldav etags for each account httpd.recent_dav_etags = {} httpd.key_shortcuts = {} load_access_keys_for_accounts(base_dir, httpd.key_shortcuts, httpd.access_keys) # wheither to use low bandwidth images httpd.low_bandwidth = low_bandwidth # list of blocked user agent types within the User-Agent header httpd.user_agents_blocked = user_agents_blocked # list of crawler bots permitted within the User-Agent header httpd.crawlers_allowed = crawlers_allowed # list of web crawlers known to the system httpd.known_bots = load_known_web_bots(base_dir) httpd.unit_test = unit_test httpd.allow_local_network_access = allow_local_network_access if unit_test: # unit tests are run on the local network with LAN addresses httpd.allow_local_network_access = True httpd.yt_replace_domain = yt_replace_domain httpd.twitter_replacement_domain = twitter_replacement_domain # newswire storing rss feeds httpd.newswire = {} # maximum number of posts to appear in the newswire on the right column httpd.max_newswire_posts = max_newswire_posts # whether to require that all incoming posts have valid jsonld signatures httpd.verify_all_signatures = verify_all_signatures # This counter is used to update the list of blocked domains in memory. # It helps to avoid touching the disk and so improves flooding resistance httpd.blocklistUpdateCtr = 0 httpd.blocklistUpdateInterval = 100 httpd.domainBlocklist = get_domain_blocklist(base_dir) httpd.manual_follower_approval = manual_follower_approval if domain.endswith('.onion'): onion_domain = domain elif domain.endswith('.i2p'): i2p_domain = domain httpd.onion_domain = onion_domain httpd.i2p_domain = i2p_domain httpd.media_instance = media_instance httpd.blogs_instance = blogs_instance # load translations dictionary httpd.translate = {} httpd.system_language = 'en' if not unit_test: httpd.translate, httpd.system_language = \ load_translations_from_file(base_dir, language) if not httpd.system_language: print('ERROR: no system language loaded') sys.exit() print('System language: ' + httpd.system_language) if not httpd.translate: print('ERROR: no translations were loaded') sys.exit() # create hashtag categories for cities load_city_hashtags(base_dir, httpd.translate) # spoofed city for gps location misdirection httpd.city = city # For moderated newswire feeds this is the amount of time allowed # for voting after the post arrives httpd.voting_time_mins = voting_time_mins # on the newswire, whether moderators vote positively for items # or against them (veto) httpd.positive_voting = positive_voting # number of votes needed to remove a newswire item from the news timeline # or if positive voting is anabled to add the item to the news timeline httpd.newswire_votes_threshold = newswire_votes_threshold # maximum overall size of an rss/atom feed read by the newswire daemon # If the feed is too large then this is probably a DoS attempt httpd.max_newswire_feed_size_kb = max_newswire_feed_size_kb # For each newswire source (account or rss feed) # this is the maximum number of posts to show for each. # This avoids one or two sources from dominating the news, # and also prevents big feeds from slowing down page load times httpd.max_newswire_posts_per_source = max_newswire_posts_per_source # Show only the date at the bottom of posts, and not the time httpd.show_published_date_only = show_published_date_only # maximum number of news articles to mirror httpd.max_mirrored_articles = max_mirrored_articles # maximum number of posts in the news timeline/outbox httpd.max_news_posts = max_news_posts # The maximum number of tags per post which can be # attached to RSS feeds pulled in via the newswire httpd.maxTags = 32 # maximum number of followers per account httpd.max_followers = max_followers # whether to show an icon for publish on the # newswire, or a 'Publish' button httpd.show_publish_as_icon = show_publish_as_icon # Whether to show the timeline header containing inbox, outbox # calendar, etc as the full width of the screen or not httpd.full_width_tl_button_header = full_width_tl_button_header # whether to show icons in the header (eg calendar) as buttons httpd.icons_as_buttons = icons_as_buttons # whether to show the RSS icon at the top or the bottom of the timeline httpd.rss_icon_at_top = rss_icon_at_top # Whether to show the newswire publish button at the top, # above the header image httpd.publish_button_at_top = publish_button_at_top # maximum size of individual RSS feed items, in K httpd.max_feed_item_size_kb = max_feed_item_size_kb # maximum size of a hashtag category, in K httpd.maxCategoriesFeedItemSizeKb = 1024 # how many months does a followed account need to be unseen # for it to be considered dormant? httpd.dormant_months = dormant_months # maximum number of likes to display on a post httpd.max_like_count = max_like_count if httpd.max_like_count < 0: httpd.max_like_count = 0 elif httpd.max_like_count > 16: httpd.max_like_count = 16 httpd.followingItemsPerPage = 12 if registration == 'open': httpd.registration = True else: httpd.registration = False httpd.enable_shared_inbox = enable_shared_inbox httpd.outboxThread = {} httpd.outbox_thread_index = {} httpd.new_post_thread = {} httpd.project_version = project_version httpd.secure_mode = secure_mode # max POST size of 30M httpd.max_post_length = 1024 * 1024 * 30 httpd.maxMediaSize = httpd.max_post_length # Maximum text length is 64K - enough for a blog post httpd.maxMessageLength = 64000 # Maximum overall number of posts per box httpd.maxPostsInBox = 32000 httpd.maxCacheAgeDays = 30 httpd.domain = domain httpd.port = port httpd.domain_full = get_full_domain(domain, port) if onion_domain: save_domain_qrcode(base_dir, 'http', onion_domain) elif i2p_domain: save_domain_qrcode(base_dir, 'http', i2p_domain) else: save_domain_qrcode(base_dir, http_prefix, httpd.domain_full) clear_person_qrcodes(base_dir) httpd.http_prefix = http_prefix httpd.debug = debug httpd.federation_list = fed_list.copy() httpd.shared_items_federated_domains = \ shared_items_federated_domains.copy() httpd.base_dir = base_dir httpd.instance_id = instance_id httpd.person_cache = {} httpd.cached_webfingers = {} httpd.favicons_cache = {} httpd.proxy_type = proxy_type httpd.session = None httpd.session_onion = None httpd.session_i2p = None httpd.last_getreq = 0 httpd.last_postreq = 0 httpd.getreq_busy = False httpd.postreq_busy = False httpd.received_message = False httpd.inbox_queue = [] httpd.send_threads = send_threads httpd.postLog = [] httpd.max_queue_length = 64 httpd.allow_deletion = allow_deletion httpd.last_login_time = 0 httpd.last_login_failure = 0 httpd.login_failure_count = {} httpd.log_login_failures = log_login_failures httpd.max_replies = max_replies httpd.tokens = {} httpd.tokens_lookup = {} load_tokens(base_dir, httpd.tokens, httpd.tokens_lookup) httpd.instance_only_skills_search = instance_only_skills_search # contains threads used to send posts to followers httpd.followers_threads = [] # create a cache of blocked domains in memory. # This limits the amount of slow disk reads which need to be done httpd.blocked_cache = [] httpd.blocked_cache_last_updated = 0 httpd.blocked_cache_update_secs = 120 httpd.blocked_cache_last_updated = \ update_blocked_cache(base_dir, httpd.blocked_cache, httpd.blocked_cache_last_updated, 0) # get the list of custom emoji, for use by the mastodon api httpd.custom_emoji = \ metadata_custom_emoji(base_dir, http_prefix, httpd.domain_full) # whether to enable broch mode, which locks down the instance set_broch_mode(base_dir, httpd.domain_full, broch_mode) if not os.path.isdir(base_dir + '/accounts/inbox@' + domain): print('Creating shared inbox: inbox@' + domain) create_shared_inbox(base_dir, 'inbox', domain, port, http_prefix) if not os.path.isdir(base_dir + '/accounts/news@' + domain): print('Creating news inbox: news@' + domain) create_news_inbox(base_dir, domain, port, http_prefix) set_config_param(base_dir, "listsEnabled", "Murdoch press") # dict of known web crawlers accessing nodeinfo or the masto API # and how many times they have been seen httpd.known_crawlers = {} known_crawlers_filename = base_dir + '/accounts/knownCrawlers.json' if os.path.isfile(known_crawlers_filename): httpd.known_crawlers = load_json(known_crawlers_filename) # when was the last crawler seen? httpd.last_known_crawler = 0 if lists_enabled: httpd.lists_enabled = lists_enabled else: httpd.lists_enabled = get_config_param(base_dir, "listsEnabled") httpd.cw_lists = load_cw_lists(base_dir, True) # set the avatar for the news account httpd.theme_name = get_config_param(base_dir, 'theme') if not httpd.theme_name: httpd.theme_name = 'default' if is_news_theme_name(base_dir, httpd.theme_name): news_instance = True httpd.news_instance = news_instance httpd.default_timeline = 'inbox' if media_instance: httpd.default_timeline = 'tlmedia' if blogs_instance: httpd.default_timeline = 'tlblogs' if news_instance: httpd.default_timeline = 'tlfeatures' set_news_avatar(base_dir, httpd.theme_name, http_prefix, domain, httpd.domain_full) if not os.path.isdir(base_dir + '/cache'): os.mkdir(base_dir + '/cache') if not os.path.isdir(base_dir + '/cache/actors'): print('Creating actors cache') os.mkdir(base_dir + '/cache/actors') if not os.path.isdir(base_dir + '/cache/announce'): print('Creating announce cache') os.mkdir(base_dir + '/cache/announce') if not os.path.isdir(base_dir + '/cache/avatars'): print('Creating avatars cache') os.mkdir(base_dir + '/cache/avatars') archive_dir = base_dir + '/archive' if not os.path.isdir(archive_dir): print('Creating archive') os.mkdir(archive_dir) if not os.path.isdir(base_dir + '/sharefiles'): print('Creating shared item files directory') os.mkdir(base_dir + '/sharefiles') print('THREAD: Creating fitness thread') httpd.thrFitness = \ thread_with_trace(target=fitness_thread, args=(base_dir, httpd.fitness), daemon=True) begin_thread(httpd.thrFitness, 'run_daemon thrFitness') httpd.recent_posts_cache = {} print('THREAD: Creating cache expiry thread') httpd.thrCache = \ thread_with_trace(target=expire_cache, args=(base_dir, httpd.person_cache, httpd.http_prefix, archive_dir, httpd.recent_posts_cache, httpd.maxPostsInBox, httpd.maxCacheAgeDays), daemon=True) begin_thread(httpd.thrCache, 'run_daemon thrCache') # number of mins after which sending posts or updates will expire httpd.send_threads_timeout_mins = send_threads_timeout_mins print('THREAD: Creating posts queue') httpd.thrPostsQueue = \ thread_with_trace(target=run_posts_queue, args=(base_dir, httpd.send_threads, debug, httpd.send_threads_timeout_mins), daemon=True) if not unit_test: print('THREAD: run_posts_watchdog') httpd.thrPostsWatchdog = \ thread_with_trace(target=run_posts_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrPostsWatchdog, 'run_daemon thrPostWatchdog') else: begin_thread(httpd.thrPostsQueue, 'run_daemon thrPostWatchdog 2') print('THREAD: Creating expire thread for shared items') httpd.thrSharesExpire = \ thread_with_trace(target=run_shares_expire, args=(project_version, base_dir, httpd), daemon=True) if not unit_test: print('THREAD: run_shares_expire_watchdog') httpd.thrSharesExpireWatchdog = \ thread_with_trace(target=run_shares_expire_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrSharesExpireWatchdog, 'run_daemon thrSharesExpireWatchdog') else: begin_thread(httpd.thrSharesExpire, 'run_daemon thrSharesExpireWatchdog 2') httpd.max_recent_posts = max_recent_posts httpd.iconsCache = {} httpd.fontsCache = {} # create tokens used for shared item federation fed_domains = httpd.shared_items_federated_domains httpd.shared_item_federation_tokens = \ generate_shared_item_federation_tokens(fed_domains, base_dir) si_federation_tokens = httpd.shared_item_federation_tokens httpd.shared_item_federation_tokens = \ create_shared_item_federation_token(base_dir, httpd.domain_full, False, si_federation_tokens) # load peertube instances from file into a list httpd.peertube_instances = [] load_peertube_instances(base_dir, httpd.peertube_instances) create_initial_last_seen(base_dir, http_prefix) httpd.max_mentions = max_mentions httpd.max_emoji = max_emoji httpd.max_hashtags = max_hashtags print('THREAD: Creating inbox queue') httpd.thrInboxQueue = \ thread_with_trace(target=run_inbox_queue, args=(httpd, httpd.recent_posts_cache, httpd.max_recent_posts, project_version, base_dir, http_prefix, httpd.send_threads, httpd.postLog, httpd.cached_webfingers, httpd.person_cache, httpd.inbox_queue, domain, onion_domain, i2p_domain, port, proxy_type, httpd.federation_list, max_replies, domain_max_posts_per_day, account_max_posts_per_day, allow_deletion, debug, max_mentions, max_emoji, httpd.translate, unit_test, httpd.yt_replace_domain, httpd.twitter_replacement_domain, httpd.show_published_date_only, httpd.max_followers, httpd.allow_local_network_access, httpd.peertube_instances, verify_all_signatures, httpd.theme_name, httpd.system_language, httpd.max_like_count, httpd.signing_priv_key_pem, httpd.default_reply_interval_hrs, httpd.cw_lists, httpd.max_hashtags), daemon=True) print('THREAD: Creating scheduled post thread') httpd.thrPostSchedule = \ thread_with_trace(target=run_post_schedule, args=(base_dir, httpd, 20), daemon=True) print('THREAD: Creating newswire thread') httpd.thrNewswireDaemon = \ thread_with_trace(target=run_newswire_daemon, args=(base_dir, httpd, http_prefix, domain, port, httpd.translate), daemon=True) print('THREAD: Creating federated shares thread') httpd.thrFederatedSharesDaemon = \ thread_with_trace(target=run_federated_shares_daemon, args=(base_dir, httpd, http_prefix, httpd.domain_full, proxy_type, debug, httpd.system_language), daemon=True) # flags used when restarting the inbox queue httpd.restart_inbox_queue_in_progress = False httpd.restart_inbox_queue = False update_hashtag_categories(base_dir) print('Adding hashtag categories for language ' + httpd.system_language) load_hashtag_categories(base_dir, httpd.system_language) # signing key used for authorized fetch # this is the instance actor private key httpd.signing_priv_key_pem = get_instance_actor_key(base_dir, domain) # threads used for checking for actor changes when clicking on # avatar icon / person options httpd.thrCheckActor = {} if not unit_test: print('THREAD: Creating import following watchdog') httpd.thrImportFollowing = \ thread_with_trace(target=run_import_following_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrImportFollowing, 'run_daemon thrImportFollowing') print('THREAD: Creating inbox queue watchdog') httpd.thrWatchdog = \ thread_with_trace(target=run_inbox_queue_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrWatchdog, 'run_daemon thrWatchdog') print('THREAD: Creating scheduled post watchdog') httpd.thrWatchdogSchedule = \ thread_with_trace(target=run_post_schedule_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrWatchdogSchedule, 'run_daemon thrWatchdogSchedule') print('THREAD: Creating newswire watchdog') httpd.thrNewswireWatchdog = \ thread_with_trace(target=run_newswire_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrNewswireWatchdog, 'run_daemon thrNewswireWatchdog') print('THREAD: Creating federated shares watchdog') httpd.thrFederatedSharesWatchdog = \ thread_with_trace(target=run_federated_shares_watchdog, args=(project_version, httpd), daemon=True) begin_thread(httpd.thrFederatedSharesWatchdog, 'run_daemon thrFederatedSharesWatchdog') print('THREAD: Creating federated blocks thread') httpd.thrFederatedBlocksDaemon = \ thread_with_trace(target=run_federated_blocks_daemon, args=(base_dir, httpd, debug), daemon=True) begin_thread(httpd.thrFederatedBlocksDaemon, 'run_daemon thrFederatedBlocksDaemon') else: print('Starting inbox queue') begin_thread(httpd.thrInboxQueue, 'run_daemon start inbox') print('Starting scheduled posts daemon') begin_thread(httpd.thrPostSchedule, 'run_daemon start scheduled posts') print('Starting federated shares daemon') begin_thread(httpd.thrFederatedSharesDaemon, 'run_daemon start federated shares') update_memorial_flags(base_dir, httpd.person_cache) if client_to_server: print('Running ActivityPub client on ' + domain + ' port ' + str(proxy_port)) else: print('Running ActivityPub server on ' + domain + ' port ' + str(proxy_port)) httpd.starting_daemon = False httpd.serve_forever()