diff --git a/acceptreject.py b/acceptreject.py index 054350110..1ce9d256c 100644 --- a/acceptreject.py +++ b/acceptreject.py @@ -76,7 +76,9 @@ def create_reject(base_dir: str, federation_list: [], def _accept_follow(base_dir: str, domain: str, message_json: {}, - federation_list: [], debug: bool) -> None: + federation_list: [], debug: bool, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> None: """Receiving a follow Accept activity """ if not has_object_stringType(message_json, debug): @@ -144,6 +146,16 @@ def _accept_follow(base_dir: str, domain: str, message_json: {}, followed_actor) return + # convert from onion/i2p to clearnet accepted domain + if onion_domain: + if accepted_domain.endswith('.onion') and \ + not curr_domain.endswith('.onion'): + accepted_domain = curr_domain + if i2p_domain: + if accepted_domain.endswith('.i2p') and \ + not curr_domain.endswith('.i2p'): + accepted_domain = curr_domain + accepted_domain_full = accepted_domain if accepted_port: accepted_domain_full = accepted_domain + ':' + str(accepted_port) @@ -189,7 +201,9 @@ def receive_accept_reject(session, base_dir: str, cached_webfingers: {}, person_cache: {}, message_json: {}, federation_list: [], - debug: bool) -> bool: + debug: bool, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> bool: """Receives an Accept or Reject within the POST section of HTTPServer """ if message_json['type'] != 'Accept' and message_json['type'] != 'Reject': @@ -215,7 +229,8 @@ def receive_accept_reject(session, base_dir: str, ' does not contain a nickname. ' + 'Assuming single user instance.') # receive follow accept - _accept_follow(base_dir, domain, message_json, federation_list, debug) + _accept_follow(base_dir, domain, message_json, federation_list, debug, + curr_domain, onion_domain, i2p_domain) if debug: print('DEBUG: Uh, ' + message_json['type'] + ', I guess') return True diff --git a/announce.py b/announce.py index 5538b72be..f4c041f93 100644 --- a/announce.py +++ b/announce.py @@ -141,7 +141,9 @@ def create_announce(session, base_dir: str, federation_list: [], send_threads: [], postLog: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> {}: """Creates an announce message Typically to_url will be https://www.w3.org/ns/activitystreams#Public and ccUrl might be a specific person favorited or repeated and the @@ -202,7 +204,8 @@ def create_announce(session, base_dir: str, federation_list: [], send_threads, postLog, cached_webfingers, person_cache, debug, project_version, None, group_account, - signing_priv_key_pem, 639633) + signing_priv_key_pem, 639633, + curr_domain, onion_domain, i2p_domain) return new_announce @@ -213,7 +216,9 @@ def announce_public(session, base_dir: str, federation_list: [], send_threads: [], postLog: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> {}: """Makes a public announcement """ from_domain = get_full_domain(domain, port) @@ -227,7 +232,8 @@ def announce_public(session, base_dir: str, federation_list: [], send_threads, postLog, person_cache, cached_webfingers, debug, project_version, - signing_priv_key_pem) + signing_priv_key_pem, curr_domain, + onion_domain, i2p_domain) def send_announce_via_server(base_dir: str, session, diff --git a/daemon.py b/daemon.py index 60e845781..110ced53f 100644 --- a/daemon.py +++ b/daemon.py @@ -1479,7 +1479,9 @@ class PubServer(BaseHTTPRequestHandler): str(self.server.base_dir)) wf_result = \ webfinger_lookup(self.path, self.server.base_dir, - self.server.domain, self.server.onion_domain, + self.server.domain, + self.server.onion_domain, + self.server.i2p_domain, self.server.port, self.server.debug) if wf_result: msg_str = json.dumps(wf_result) @@ -1605,6 +1607,7 @@ class PubServer(BaseHTTPRequestHandler): print('Creating outbox thread ' + account_outbox_thread_name + '/' + str(self.server.outbox_thread_index[account_outbox_thread_name])) + print('THREAD: _post_to_outbox') self.server.outboxThread[account_outbox_thread_name][index] = \ thread_with_trace(target=self._post_to_outbox, args=(message_json.copy(), @@ -3374,18 +3377,30 @@ class PubServer(BaseHTTPRequestHandler): curr_domain = domain curr_port = port curr_http_prefix = http_prefix + curr_proxy_type = proxy_type if onion_domain: - if following_domain.endswith('.onion'): + if not curr_domain.endswith('.onion') and \ + following_domain.endswith('.onion'): curr_session = self.server.session_onion curr_domain = onion_domain curr_port = 80 + following_port = 80 curr_http_prefix = 'http' + curr_proxy_type = 'tor' if i2p_domain: - if following_domain.endswith('.i2p'): + if not curr_domain.endswith('.i2p') and \ + following_domain.endswith('.i2p'): curr_session = self.server.session_i2p curr_domain = i2p_domain curr_port = 80 + following_port = 80 curr_http_prefix = 'http' + curr_proxy_type = 'i2p' + + curr_session = \ + self._establish_session("follow request", + curr_session, + curr_proxy_type) send_follow_request(curr_session, base_dir, follower_nickname, @@ -3401,7 +3416,10 @@ class PubServer(BaseHTTPRequestHandler): self.server.cached_webfingers, self.server.person_cache, debug, self.server.project_version, - self.server.signing_priv_key_pem) + self.server.signing_priv_key_pem, + self.server.domain, + self.server.onion_domain, + self.server.i2p_domain) if calling_domain.endswith('.onion') and onion_domain: origin_path_str = 'http://' + onion_domain + users_path elif (calling_domain.endswith('.i2p') and i2p_domain): @@ -8295,7 +8313,10 @@ class PubServer(BaseHTTPRequestHandler): self.server.cached_webfingers, debug, self.server.project_version, - self.server.signing_priv_key_pem) + self.server.signing_priv_key_pem, + self.server.domain, + self.server.onion_domain, + self.server.i2p_domain) announce_filename = None if announce_json: # save the announce straight to the outbox @@ -8519,25 +8540,28 @@ class PubServer(BaseHTTPRequestHandler): handle_nickname + '@' + \ get_full_domain(handle_domain, handle_port) if '@' in following_handle: - if self.server.onion_domain: if following_handle.endswith('.onion'): curr_session = self.server.session_onion proxy_type = 'tor' + port = 80 if self.server.i2p_domain: if following_handle.endswith('.i2p'): curr_session = self.server.session_i2p proxy_type = 'i2p' + port = 80 curr_session = \ - self._establish_session("followApproveButton", + self._establish_session("follow_approve_button", curr_session, proxy_type) if not curr_session: + print('WARN: unable to establish session ' + + 'when approving follow request') self._404() return signing_priv_key_pem = \ self.server.signing_priv_key_pem - manual_approve_follow_request_thread(curr_session, + manual_approve_follow_request_thread(self.server.session, self.server.session_onion, self.server.session_i2p, self.server.onion_domain, @@ -8553,7 +8577,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.person_cache, debug, self.server.project_version, - signing_priv_key_pem) + signing_priv_key_pem, + proxy_type) origin_path_str_absolute = \ http_prefix + '://' + domain_full + origin_path_str if calling_domain.endswith('.onion') and onion_domain: @@ -8701,7 +8726,7 @@ class PubServer(BaseHTTPRequestHandler): handle_nickname + '@' + \ get_full_domain(handle_domain, handle_port) if '@' in following_handle: - manual_deny_follow_request_thread(curr_session, + manual_deny_follow_request_thread(self.server.session, self.server.session_onion, self.server.session_i2p, self.server.onion_domain, @@ -20184,7 +20209,7 @@ def run_shares_expire(version_number: str, base_dir: str) -> None: def run_posts_watchdog(project_version: str, httpd) -> None: """This tries to keep the posts thread running even if it dies """ - print('Starting posts queue watchdog') + print('THREAD: Starting posts queue watchdog') posts_queue_original = httpd.thrPostsQueue.clone(run_posts_queue) httpd.thrPostsQueue.start() while True: @@ -20192,6 +20217,7 @@ def run_posts_watchdog(project_version: str, httpd) -> None: if httpd.thrPostsQueue.is_alive(): continue httpd.thrPostsQueue.kill() + print('THREAD: restarting posts queue') httpd.thrPostsQueue = posts_queue_original.clone(run_posts_queue) httpd.thrPostsQueue.start() print('Restarting posts queue...') @@ -20200,7 +20226,7 @@ def run_posts_watchdog(project_version: str, httpd) -> None: def run_shares_expire_watchdog(project_version: str, httpd) -> None: """This tries to keep the shares expiry thread running even if it dies """ - print('Starting shares expiry watchdog') + print('THREAD: Starting shares expiry watchdog') shares_expire_original = httpd.thrSharesExpire.clone(run_shares_expire) httpd.thrSharesExpire.start() while True: @@ -20208,6 +20234,7 @@ def run_shares_expire_watchdog(project_version: str, httpd) -> None: if httpd.thrSharesExpire.is_alive(): continue httpd.thrSharesExpire.kill() + print('THREAD: restarting shares watchdog') httpd.thrSharesExpire = shares_expire_original.clone(run_shares_expire) httpd.thrSharesExpire.start() print('Restarting shares expiry...') @@ -20688,13 +20715,13 @@ def run_daemon(crawlers_allowed: [], print('Creating shared item files directory') os.mkdir(base_dir + '/sharefiles') - print('Creating fitness thread') + print('THREAD: Creating fitness thread') httpd.thrFitness = \ thread_with_trace(target=fitness_thread, args=(base_dir, httpd.fitness), daemon=True) httpd.thrFitness.start() - print('Creating cache expiry thread') + print('THREAD: Creating cache expiry thread') httpd.thrCache = \ thread_with_trace(target=expire_cache, args=(base_dir, httpd.person_cache, @@ -20706,12 +20733,13 @@ def run_daemon(crawlers_allowed: [], # number of mins after which sending posts or updates will expire httpd.send_threads_timeout_mins = send_threads_timeout_mins - print('Creating posts queue') + print('THREAD: Creating posts queue') httpd.thrPostsQueue = \ thread_with_trace(target=run_posts_queue, args=(base_dir, httpd.send_threads, debug, httpd.send_threads_timeout_mins), daemon=True) if not unit_test: + print('THREAD: run_posts_watchdog') httpd.thrPostsWatchdog = \ thread_with_trace(target=run_posts_watchdog, args=(project_version, httpd), daemon=True) @@ -20719,11 +20747,12 @@ def run_daemon(crawlers_allowed: [], else: httpd.thrPostsQueue.start() - print('Creating expire thread for shared items') + print('THREAD: Creating expire thread for shared items') httpd.thrSharesExpire = \ thread_with_trace(target=run_shares_expire, args=(project_version, base_dir), daemon=True) if not unit_test: + print('THREAD: run_shares_expire_watchdog') httpd.thrSharesExpireWatchdog = \ thread_with_trace(target=run_shares_expire_watchdog, args=(project_version, httpd), daemon=True) @@ -20752,10 +20781,10 @@ def run_daemon(crawlers_allowed: [], create_initial_last_seen(base_dir, http_prefix) - print('Creating inbox queue') + print('THREAD: Creating inbox queue') httpd.thrInboxQueue = \ thread_with_trace(target=run_inbox_queue, - args=(httpd.recent_posts_cache, + args=(httpd, httpd.recent_posts_cache, httpd.max_recent_posts, project_version, base_dir, http_prefix, httpd.send_threads, @@ -20784,19 +20813,19 @@ def run_daemon(crawlers_allowed: [], httpd.default_reply_interval_hrs, httpd.cw_lists), daemon=True) - print('Creating scheduled post thread') + print('THREAD: Creating scheduled post thread') httpd.thrPostSchedule = \ thread_with_trace(target=run_post_schedule, args=(base_dir, httpd, 20), daemon=True) - print('Creating newswire thread') + print('THREAD: Creating newswire thread') httpd.thrNewswireDaemon = \ thread_with_trace(target=run_newswire_daemon, args=(base_dir, httpd, http_prefix, domain, port, httpd.translate), daemon=True) - print('Creating federated shares thread') + print('THREAD: Creating federated shares thread') httpd.thrFederatedSharesDaemon = \ thread_with_trace(target=run_federated_shares_daemon, args=(base_dir, httpd, @@ -20818,25 +20847,25 @@ def run_daemon(crawlers_allowed: [], httpd.signing_priv_key_pem = get_instance_actor_key(base_dir, domain) if not unit_test: - print('Creating inbox queue watchdog') + print('THREAD: Creating inbox queue watchdog') httpd.thrWatchdog = \ thread_with_trace(target=run_inbox_queue_watchdog, args=(project_version, httpd), daemon=True) httpd.thrWatchdog.start() - print('Creating scheduled post watchdog') + print('THREAD: Creating scheduled post watchdog') httpd.thrWatchdogSchedule = \ thread_with_trace(target=run_post_schedule_watchdog, args=(project_version, httpd), daemon=True) httpd.thrWatchdogSchedule.start() - print('Creating newswire watchdog') + print('THREAD: Creating newswire watchdog') httpd.thrNewswireWatchdog = \ thread_with_trace(target=run_newswire_watchdog, args=(project_version, httpd), daemon=True) httpd.thrNewswireWatchdog.start() - print('Creating federated shares watchdog') + print('THREAD: Creating federated shares watchdog') httpd.thrFederatedSharesWatchdog = \ thread_with_trace(target=run_federated_shares_watchdog, args=(project_version, httpd), daemon=True) diff --git a/desktop_client.py b/desktop_client.py index ba4d4bdf3..35d83f535 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -38,7 +38,7 @@ from follow import deny_follow_request_via_server from follow import get_follow_requests_via_server from follow import get_following_via_server from follow import get_followers_via_server -from follow import send_follow_requestViaServer +from follow import send_follow_request_via_server from follow import send_unfollow_request_via_server from posts import send_block_via_server from posts import send_undo_block_via_server @@ -2265,18 +2265,18 @@ def run_desktop_client(base_dir: str, proxy_type: str, http_prefix: str, _say_command(say_str, say_str, screenreader, system_language, espeak) session_follow = create_session(proxy_type) - send_follow_requestViaServer(base_dir, - session_follow, - nickname, password, - domain, port, - follow_nickname, - follow_domain, - follow_port, - http_prefix, - cached_webfingers, - person_cache, - debug, __version__, - signing_priv_key_pem) + send_follow_request_via_server(base_dir, + session_follow, + nickname, password, + domain, port, + follow_nickname, + follow_domain, + follow_port, + http_prefix, + cached_webfingers, + person_cache, + debug, __version__, + signing_priv_key_pem) else: if follow_handle: say_str = follow_handle + ' is not valid' diff --git a/epicyon.py b/epicyon.py index 57a62f784..bd4fca62f 100644 --- a/epicyon.py +++ b/epicyon.py @@ -56,7 +56,7 @@ from follow import get_following_via_server from follow import get_followers_via_server from follow import clear_follows from follow import add_follower_of_person -from follow import send_follow_requestViaServer +from follow import send_follow_request_via_server from follow import send_unfollow_request_via_server from tests import test_shared_items_federation from tests import test_group_follow @@ -1293,7 +1293,7 @@ if args.approve: send_threads, postLog, cached_webfingers, person_cache, debug, __version__, - signing_priv_key_pem) + signing_priv_key_pem, proxy_type) sys.exit() if args.deny: @@ -2067,13 +2067,13 @@ if args.follow: if args.secure_mode: signing_priv_key_pem = get_instance_actor_key(base_dir, domain) - send_follow_requestViaServer(base_dir, session, - args.nickname, args.password, - domain, port, - follow_nickname, follow_domain, follow_port, - follow_http_prefix, - cached_webfingers, person_cache, - debug, __version__, signing_priv_key_pem) + send_follow_request_via_server(base_dir, session, + args.nickname, args.password, + domain, port, + follow_nickname, follow_domain, follow_port, + follow_http_prefix, + cached_webfingers, person_cache, + debug, __version__, signing_priv_key_pem) for t in range(20): time.sleep(1) # TODO some method to know if it worked diff --git a/follow.py b/follow.py index 3349e3b0b..cfd234ddd 100644 --- a/follow.py +++ b/follow.py @@ -108,8 +108,8 @@ def _remove_from_follow_base(base_dir: str, approve_follows_filename = accounts_dir + '/' + follow_file + '.txt' if not os.path.isfile(approve_follows_filename): if debug: - print('WARN: Approve follow requests file ' + - approve_follows_filename + ' not found') + print('There is no ' + follow_file + + ' to remove ' + handle + ' from') return accept_deny_actor = None if accept_or_deny_handle not in open(approve_follows_filename).read(): @@ -731,7 +731,9 @@ def followed_account_accepts(session, base_dir: str, http_prefix: str, cached_webfingers: {}, person_cache: {}, debug: bool, project_version: str, removeFollowActivity: bool, - signing_priv_key_pem: str): + signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str): """The person receiving a follow request accepts the new follower and sends back an Accept activity """ @@ -780,10 +782,12 @@ def followed_account_accepts(session, base_dir: str, http_prefix: str, send_threads, postLog, cached_webfingers, person_cache, debug, project_version, None, group_account, signing_priv_key_pem, - 7856837) + 7856837, curr_domain, onion_domain, i2p_domain) -def followed_account_rejects(session, base_dir: str, http_prefix: str, +def followed_account_rejects(session, session_onion, session_i2p, + onion_domain: str, i2p_domain: str, + base_dir: str, http_prefix: str, nickname_to_follow: str, domain_to_follow: str, port: int, nickname: str, domain: str, from_port: int, @@ -848,7 +852,8 @@ def followed_account_rejects(session, base_dir: str, http_prefix: str, send_threads, postLog, cached_webfingers, person_cache, debug, project_version, None, group_account, signing_priv_key_pem, - 6393063) + 6393063, + domain, onion_domain, i2p_domain) def send_follow_request(session, base_dir: str, @@ -861,7 +866,9 @@ def send_follow_request(session, base_dir: str, client_to_server: bool, federation_list: [], send_threads: [], postLog: [], cached_webfingers: {}, person_cache: {}, debug: bool, - project_version: str, signing_priv_key_pem: str) -> {}: + project_version: str, signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> {}: """Gets the json object for sending a follow request """ if not signing_priv_key_pem: @@ -942,24 +949,25 @@ def send_follow_request(session, base_dir: str, federation_list, send_threads, postLog, cached_webfingers, person_cache, debug, project_version, None, group_account, - signing_priv_key_pem, 8234389) + signing_priv_key_pem, 8234389, + curr_domain, onion_domain, i2p_domain) return new_follow_json -def send_follow_requestViaServer(base_dir: str, session, - from_nickname: str, password: str, - from_domain: str, from_port: int, - follow_nickname: str, follow_domain: str, - followPort: int, - http_prefix: str, - cached_webfingers: {}, person_cache: {}, - debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: +def send_follow_request_via_server(base_dir: str, session, + from_nickname: str, password: str, + from_domain: str, from_port: int, + follow_nickname: str, follow_domain: str, + followPort: int, + http_prefix: str, + cached_webfingers: {}, person_cache: {}, + debug: bool, project_version: str, + signing_priv_key_pem: str) -> {}: """Creates a follow request via c2s """ if not session: - print('WARN: No session for send_follow_requestViaServer') + print('WARN: No session for send_follow_request_via_server') return 6 from_domain_full = get_full_domain(from_domain, from_port) diff --git a/inbox.py b/inbox.py index 1f83888e7..b3ba36255 100644 --- a/inbox.py +++ b/inbox.py @@ -606,7 +606,8 @@ def save_post_to_inbox_queue(base_dir: str, http_prefix: str, def _inbox_post_recipients_add(base_dir: str, http_prefix: str, toList: [], recipients_dict: {}, domain_match: str, domain: str, - actor: str, debug: bool) -> bool: + actor: str, debug: bool, + onion_domain: str, i2p_domain: str) -> bool: """Given a list of post recipients (toList) from 'to' or 'cc' parameters populate a recipients_dict with the handle for each """ @@ -614,9 +615,18 @@ def _inbox_post_recipients_add(base_dir: str, http_prefix: str, toList: [], for recipient in toList: if not recipient: continue - # is this a to a local account? + # if the recipient is an onion or i2p address then + # is it an account on a clearnet instance? + # If so then change the onion/i2p to the account domain + if onion_domain: + if onion_domain + '/' in recipient: + recipient = recipient.replace(onion_domain, domain) + if i2p_domain: + if i2p_domain + '/' in recipient: + recipient = recipient.replace(i2p_domain, domain) + # is this a to an account on this instance? if domain_match in recipient: - # get the handle for the local account + # get the handle for the account on this instance nickname = recipient.split(domain_match)[1] handle = nickname + '@' + domain if os.path.isdir(base_dir + '/accounts/' + handle): @@ -643,7 +653,8 @@ def _inbox_post_recipients_add(base_dir: str, http_prefix: str, toList: [], def _inbox_post_recipients(base_dir: str, post_json_object: {}, http_prefix: str, domain: str, port: int, - debug: bool) -> ([], []): + debug: bool, + onion_domain: str, i2p_domain: str) -> ([], []): """Returns dictionaries containing the recipients of the given post The shared dictionary contains followers """ @@ -678,7 +689,8 @@ def _inbox_post_recipients(base_dir: str, post_json_object: {}, recipients_list, recipients_dict, domain_match, domain_base, - actor, debug) + actor, debug, + onion_domain, i2p_domain) if includes_followers: follower_recipients = True else: @@ -695,7 +707,8 @@ def _inbox_post_recipients(base_dir: str, post_json_object: {}, recipients_list, recipients_dict, domain_match, domain_base, - actor, debug) + actor, debug, + onion_domain, i2p_domain) if includes_followers: follower_recipients = True else: @@ -720,7 +733,8 @@ def _inbox_post_recipients(base_dir: str, post_json_object: {}, recipients_list, recipients_dict, domain_match, domain_base, - actor, debug) + actor, debug, + onion_domain, i2p_domain) if includes_followers: follower_recipients = True @@ -734,7 +748,8 @@ def _inbox_post_recipients(base_dir: str, post_json_object: {}, recipients_list, recipients_dict, domain_match, domain_base, - actor, debug) + actor, debug, + onion_domain, i2p_domain) if includes_followers: follower_recipients = True @@ -753,7 +768,8 @@ def _inbox_post_recipients(base_dir: str, post_json_object: {}, def _receive_undo_follow(session, base_dir: str, http_prefix: str, port: int, message_json: {}, federation_list: [], - debug: bool) -> bool: + debug: bool, domain: str, + onion_domain: str, i2p_domain: str) -> bool: if not message_json['object'].get('actor'): if debug: print('DEBUG: follow request has no actor within object') @@ -786,6 +802,12 @@ def _receive_undo_follow(session, base_dir: str, http_prefix: str, return False domain_following, port_following = \ get_domain_from_actor(message_json['object']['object']) + if onion_domain: + if domain_following.endswith(onion_domain): + domain_following = domain + if i2p_domain: + if domain_following.endswith(i2p_domain): + domain_following = domain domain_following_full = get_full_domain(domain_following, port_following) group_account = \ @@ -810,7 +832,8 @@ def _receive_undo(session, base_dir: str, http_prefix: str, port: int, send_threads: [], post_log: [], cached_webfingers: {}, person_cache: {}, message_json: {}, federation_list: [], - debug: bool) -> bool: + debug: bool, domain: str, + onion_domain: str, i2p_domain: str) -> bool: """Receives an undo request within the POST section of HTTPServer """ if not message_json['type'].startswith('Undo'): @@ -831,7 +854,8 @@ def _receive_undo(session, base_dir: str, http_prefix: str, message_json['object']['type'] == 'Join': return _receive_undo_follow(session, base_dir, http_prefix, port, message_json, - federation_list, debug) + federation_list, debug, + domain, onion_domain, i2p_domain) return False @@ -1009,7 +1033,7 @@ def _receive_update_activity(recent_posts_cache: {}, session, base_dir: str, def _receive_like(recent_posts_cache: {}, session, handle: str, is_group: bool, base_dir: str, http_prefix: str, domain: str, port: int, - onion_domain: str, + onion_domain: str, i2p_domain: str, send_threads: [], post_log: [], cached_webfingers: {}, person_cache: {}, message_json: {}, federation_list: [], debug: bool, @@ -2647,7 +2671,7 @@ def _group_handle(base_dir: str, handle: str) -> bool: return actor_json['type'] == 'Group' -def _send_to_group_members(session, session_onion, session_i2p, +def _send_to_group_members(server, session, session_onion, session_i2p, base_dir: str, handle: str, port: int, post_json_object: {}, http_prefix: str, federation_list: [], @@ -2655,6 +2679,7 @@ def _send_to_group_members(session, session_onion, session_i2p, cached_webfingers: {}, person_cache: {}, debug: bool, system_language: str, + curr_domain: str, onion_domain: str, i2p_domain: str, signing_priv_key_pem: str) -> None: """When a post arrives for a group send it out to the group members @@ -2707,9 +2732,10 @@ def _send_to_group_members(session, session_onion, session_i2p, http_prefix, post_id, False, False, send_threads, post_log, person_cache, cached_webfingers, - debug, __version__, signing_priv_key_pem) + debug, __version__, signing_priv_key_pem, + curr_domain, onion_domain, i2p_domain) - send_to_followers_thread(session, session_onion, session_i2p, + send_to_followers_thread(server, session, session_onion, session_i2p, base_dir, nickname, domain, onion_domain, i2p_domain, port, http_prefix, federation_list, @@ -2845,7 +2871,8 @@ def _bounce_dm(senderPostId: str, session, http_prefix: str, signing_priv_key_pem: str, content_license_url: str, languages_understood: [], - bounce_is_chat: bool) -> bool: + bounce_is_chat: bool, + curr_domain: str, onion_domain: str, i2p_domain: str) -> bool: """Sends a bounce message back to the sending handle if a DM has been rejected """ @@ -2919,7 +2946,8 @@ def _bounce_dm(senderPostId: str, session, http_prefix: str, http_prefix, False, False, federation_list, send_threads, post_log, cached_webfingers, person_cache, debug, __version__, None, group_account, - signing_priv_key_pem, 7238634) + signing_priv_key_pem, 7238634, + curr_domain, onion_domain, i2p_domain) return True @@ -2935,7 +2963,8 @@ def _is_valid_dm(base_dir: str, nickname: str, domain: str, port: int, handle: str, system_language: str, signing_priv_key_pem: str, content_license_url: str, - languages_understood: []) -> bool: + languages_understood: [], + curr_domain: str, onion_domain: str, i2p_domain: str) -> bool: """Is the given message a valid DM? """ if nickname == 'inbox': @@ -3022,7 +3051,9 @@ def _is_valid_dm(base_dir: str, nickname: str, domain: str, port: int, signing_priv_key_pem, content_license_url, languages_understood, - bounce_chat) + bounce_chat, + curr_domain, + onion_domain, i2p_domain) return False # dm index will be updated @@ -3032,7 +3063,7 @@ def _is_valid_dm(base_dir: str, nickname: str, domain: str, port: int, return True -def _receive_question_vote(base_dir: str, nickname: str, domain: str, +def _receive_question_vote(server, base_dir: str, nickname: str, domain: str, http_prefix: str, handle: str, debug: bool, post_json_object: {}, recent_posts_cache: {}, session, session_onion, session_i2p, @@ -3112,7 +3143,7 @@ def _receive_question_vote(base_dir: str, nickname: str, domain: str, question_json['type'] = 'Update' shared_items_federated_domains = [] shared_item_federation_tokens = {} - send_to_followers_thread(session, session_onion, session_i2p, + send_to_followers_thread(server, session, session_onion, session_i2p, base_dir, nickname, domain, onion_domain, i2p_domain, port, http_prefix, federation_list, @@ -3240,7 +3271,8 @@ def _check_for_git_patches(base_dir: str, nickname: str, domain: str, return 0 -def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, +def _inbox_after_initial(server, + recent_posts_cache: {}, max_recent_posts: int, session, session_onion, session_i2p, key_id: str, handle: str, message_json: {}, base_dir: str, http_prefix: str, send_threads: [], @@ -3268,6 +3300,19 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, languages_understood: []) -> bool: """ Anything which needs to be done after initial checks have passed """ + # if this is a clearnet instance then replace any onion/i2p + # domains with the account domain + if onion_domain or i2p_domain: + message_str = json.dumps(message_json, ensure_ascii=False) + if onion_domain: + if onion_domain in message_str: + message_str = message_str.replace(onion_domain, domain) + message_json = json.loads(message_str) + if i2p_domain: + if i2p_domain in message_str: + message_str = message_str.replace(i2p_domain, domain) + message_json = json.loads(message_str) + actor = key_id if '#' in actor: actor = key_id.split('#')[0] @@ -3281,7 +3326,7 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, session, handle, is_group, base_dir, http_prefix, domain, port, - onion_domain, + onion_domain, i2p_domain, send_threads, post_log, cached_webfingers, person_cache, @@ -3514,7 +3559,7 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, populate_replies(base_dir, http_prefix, domain, post_json_object, max_replies, debug) - _receive_question_vote(base_dir, nickname, domain, + _receive_question_vote(server, base_dir, nickname, domain, http_prefix, handle, debug, post_json_object, recent_posts_cache, session, session_onion, session_i2p, @@ -3550,7 +3595,9 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, handle, system_language, signing_priv_key_pem, content_license_url, - languages_understood): + languages_understood, + domain, + onion_domain, i2p_domain): return False # get the actor being replied to @@ -3684,7 +3731,8 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, # send the post out to group members if is_group: - _send_to_group_members(session, session_onion, session_i2p, + _send_to_group_members(server, + session, session_onion, session_i2p, base_dir, handle, port, post_json_object, http_prefix, federation_list, @@ -3692,7 +3740,7 @@ def _inbox_after_initial(recent_posts_cache: {}, max_recent_posts: int, post_log, cached_webfingers, person_cache, debug, system_language, - onion_domain, i2p_domain, + domain, onion_domain, i2p_domain, signing_priv_key_pem) # if the post wasn't saved @@ -3747,7 +3795,7 @@ def _restore_queue_items(base_dir: str, queue: []) -> None: def run_inbox_queue_watchdog(project_version: str, httpd) -> None: """This tries to keep the inbox thread running even if it dies """ - print('Starting inbox queue watchdog') + print('THREAD: Starting inbox queue watchdog') inbox_queue_original = httpd.thrInboxQueue.clone(run_inbox_queue) httpd.thrInboxQueue.start() while True: @@ -3755,6 +3803,7 @@ def run_inbox_queue_watchdog(project_version: str, httpd) -> None: if not httpd.thrInboxQueue.is_alive() or httpd.restart_inbox_queue: httpd.restart_inbox_queue_in_progress = True httpd.thrInboxQueue.kill() + print('THREAD: restarting inbox queue watchdog') httpd.thrInboxQueue = inbox_queue_original.clone(run_inbox_queue) httpd.inbox_queue.clear() httpd.thrInboxQueue.start() @@ -3932,7 +3981,8 @@ def _receive_follow_request(session, session_onion, session_i2p, cached_webfingers: {}, person_cache: {}, message_json: {}, federation_list: [], debug: bool, project_version: str, - max_followers: int, onion_domain: str, + max_followers: int, + this_domain: str, onion_domain: str, i2p_domain: str, signing_priv_key_pem: str, unit_test: bool) -> bool: """Receives a follow request within the POST section of HTTPServer @@ -3971,6 +4021,13 @@ def _receive_follow_request(session, session_onion, session_i2p, 'not found within object') return False domain_to_follow, temp_port = get_domain_from_actor(message_json['object']) + # switch to the local domain rather than its onion or i2p version + if onion_domain: + if domain_to_follow.endswith(onion_domain): + domain_to_follow = this_domain + if i2p_domain: + if domain_to_follow.endswith(i2p_domain): + domain_to_follow = this_domain if not domain_permitted(domain_to_follow, federation_list): if debug: print('DEBUG: follow domain not permitted ' + domain_to_follow) @@ -4002,6 +4059,7 @@ def _receive_follow_request(session, session_onion, session_i2p, base_dir + '/accounts/' + handle_to_follow) return True + is_already_follower = False if is_follower_of_person(base_dir, nickname_to_follow, domain_to_follow_full, nickname, domain_full): @@ -4009,7 +4067,7 @@ def _receive_follow_request(session, session_onion, session_i2p, print('DEBUG: ' + nickname + '@' + domain + ' is already a follower of ' + nickname_to_follow + '@' + domain_to_follow) - return True + is_already_follower = True approve_handle = nickname + '@' + domain_full @@ -4017,16 +4075,26 @@ def _receive_follow_request(session, session_onion, session_i2p, curr_http_prefix = http_prefix curr_domain = domain curr_port = from_port - if onion_domain and domain_to_follow.endswith('.onion'): + if onion_domain and \ + not curr_domain.endswith('.onion') and \ + domain_to_follow.endswith('.onion'): curr_session = session_onion curr_http_prefix = 'http' curr_domain = onion_domain curr_port = 80 - elif i2p_domain and domain_to_follow.endswith('.i2p'): + port = 80 + if debug: + print('Domain switched from ' + domain + ' to ' + curr_domain) + elif (i2p_domain and + not curr_domain.endswith('.i2p') and + domain_to_follow.endswith('.i2p')): curr_session = session_i2p curr_http_prefix = 'http' curr_domain = i2p_domain curr_port = 80 + port = 80 + if debug: + print('Domain switched from ' + domain + ' to ' + curr_domain) # is the actor sending the request valid? if not valid_sending_actor(curr_session, base_dir, @@ -4037,7 +4105,8 @@ def _receive_follow_request(session, session_onion, session_i2p, return False # what is the followers policy? - if follow_approval_required(base_dir, nickname_to_follow, + if not is_already_follower and \ + follow_approval_required(base_dir, nickname_to_follow, domain_to_follow, debug, approve_handle): print('Follow approval is required') if domain.endswith('.onion'): @@ -4089,7 +4158,12 @@ def _receive_follow_request(session, session_onion, session_i2p, message_json, debug, message_json['actor'], group_account) else: - print('Follow request does not require approval ' + approve_handle) + if is_already_follower: + print(approve_handle + ' is already a follower. ' + + 'Re-sending Accept.') + else: + print('Follow request does not require approval ' + + approve_handle) # update the followers account_to_be_followed = \ acct_dir(base_dir, nickname_to_follow, domain_to_follow) @@ -4151,6 +4225,9 @@ def _receive_follow_request(session, session_onion, session_i2p, followers_file.write(approve_handle + '\n') except OSError: print('EX: unable to write ' + followers_filename) + else: + print('ACCEPT: Follow Accept account directory not found: ' + + account_to_be_followed) print('Beginning follow accept') return followed_account_accepts(curr_session, base_dir, curr_http_prefix, @@ -4160,10 +4237,12 @@ def _receive_follow_request(session, session_onion, session_i2p, message_json, send_threads, post_log, cached_webfingers, person_cache, debug, project_version, True, - signing_priv_key_pem) + signing_priv_key_pem, + this_domain, onion_domain, i2p_domain) -def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, +def run_inbox_queue(server, + recent_posts_cache: {}, max_recent_posts: int, project_version: str, base_dir: str, http_prefix: str, send_threads: [], post_log: [], @@ -4509,7 +4588,7 @@ def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, person_cache, queue_json['post'], federation_list, - debug): + debug, domain, onion_domain, i2p_domain): print('Queue: Undo accepted from ' + key_id) if os.path.isfile(queue_filename): try: @@ -4531,7 +4610,8 @@ def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, queue_json['post'], federation_list, debug, project_version, - max_followers, onion_domain, i2p_domain, + max_followers, domain, + onion_domain, i2p_domain, signing_priv_key_pem, unit_test): if os.path.isfile(queue_filename): try: @@ -4553,7 +4633,8 @@ def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, send_threads, post_log, cached_webfingers, person_cache, queue_json['post'], - federation_list, debug): + federation_list, debug, + domain, onion_domain, i2p_domain): print('Queue: Accept/Reject received from ' + key_id) if os.path.isfile(queue_filename): try: @@ -4590,7 +4671,8 @@ def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, # get recipients list recipients_dict, recipients_dict_followers = \ _inbox_post_recipients(base_dir, queue_json['post'], - http_prefix, domain, port, debug) + http_prefix, domain, port, debug, + onion_domain, i2p_domain) if len(recipients_dict.items()) == 0 and \ len(recipients_dict_followers.items()) == 0: if debug: @@ -4646,7 +4728,8 @@ def run_inbox_queue(recent_posts_cache: {}, max_recent_posts: int, destination = \ queue_json['destination'].replace(inbox_handle, handle) languages_understood = [] - _inbox_after_initial(recent_posts_cache, + _inbox_after_initial(server, + recent_posts_cache, max_recent_posts, session, session_onion, session_i2p, key_id, handle, diff --git a/like.py b/like.py index 42b007cf8..24aebf0b7 100644 --- a/like.py +++ b/like.py @@ -77,7 +77,9 @@ def _create_like(recent_posts_cache: {}, send_threads: [], postLog: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> {}: """Creates a like actor is the person doing the liking 'to' might be a specific person (actor) whose post was liked @@ -142,7 +144,8 @@ def _create_like(recent_posts_cache: {}, send_threads, postLog, cached_webfingers, person_cache, debug, project_version, None, group_account, - signing_priv_key_pem, 7367374) + signing_priv_key_pem, 7367374, + curr_domain, onion_domain, i2p_domain) return new_like_json @@ -156,7 +159,8 @@ def like_post(recent_posts_cache: {}, send_threads: [], postLog: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, onion_domain: str, i2p_domain: str) -> {}: """Likes a given status post. This is only used by unit tests """ like_domain = get_full_domain(like_domain, like_port) @@ -170,7 +174,8 @@ def like_post(recent_posts_cache: {}, cc_list, http_prefix, object_url, actor_liked, client_to_server, send_threads, postLog, person_cache, cached_webfingers, - debug, project_version, signing_priv_key_pem) + debug, project_version, signing_priv_key_pem, + curr_domain, onion_domain, i2p_domain) def send_like_via_server(base_dir: str, session, diff --git a/manualapprove.py b/manualapprove.py index df7c7708b..2a12f7c32 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -17,6 +17,7 @@ from utils import get_port_from_domain from utils import get_user_paths from utils import acct_dir from threads import thread_with_trace +from session import create_session def manual_deny_follow_request(session, session_onion, session_i2p, @@ -88,6 +89,7 @@ def manual_deny_follow_request_thread(session, session_onion, session_i2p, """Manually deny a follow request, within a thread so that the user interface doesn't lag """ + print('THREAD: manual_deny_follow_request') thr = \ thread_with_trace(target=manual_deny_follow_request, args=(session, session_onion, session_i2p, @@ -135,7 +137,8 @@ def manual_approve_follow_request(session, session_onion, session_i2p, cached_webfingers: {}, person_cache: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> None: + signing_priv_key_pem: str, + proxy_type: str) -> None: """Manually approve a follow request """ handle = nickname + '@' + domain @@ -217,18 +220,28 @@ def manual_approve_follow_request(session, session_onion, session_i2p, curr_port = port curr_session = session curr_http_prefix = http_prefix + curr_proxy_type = proxy_type if onion_domain and \ + not curr_domain.endswith('.onion') and \ approve_domain.endswith('.onion'): curr_domain = onion_domain curr_port = 80 + approve_port = 80 curr_session = session_onion curr_http_prefix = 'http' + curr_proxy_type = 'tor' elif (i2p_domain and + not curr_domain.endswith('.i2p') and approve_domain.endswith('.i2p')): curr_domain = i2p_domain curr_port = 80 + approve_port = 80 curr_session = session_i2p curr_http_prefix = 'http' + curr_proxy_type = 'i2p' + + if not curr_session: + curr_session = create_session(curr_proxy_type) print('Manual follow accept: Sending Accept for ' + handle + ' follow request from ' + @@ -248,7 +261,10 @@ def manual_approve_follow_request(session, session_onion, session_i2p, person_cache, debug, project_version, False, - signing_priv_key_pem) + signing_priv_key_pem, + domain, + onion_domain, + i2p_domain) update_approved_followers = True else: # this isn't the approved follow so it will remain @@ -317,10 +333,12 @@ def manual_approve_follow_request_thread(session, session_onion, session_i2p, person_cache: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> None: + signing_priv_key_pem: str, + proxy_type: str) -> None: """Manually approve a follow request, in a thread so as not to cause the UI to lag """ + print('THREAD: manual_approve_follow_request') thr = \ thread_with_trace(target=manual_approve_follow_request, args=(session, session_onion, session_i2p, @@ -333,6 +351,7 @@ def manual_approve_follow_request_thread(session, session_onion, session_i2p, cached_webfingers, person_cache, debug, project_version, - signing_priv_key_pem), daemon=True) + signing_priv_key_pem, + proxy_type), daemon=True) thr.start() send_threads.append(thr) diff --git a/newsdaemon.py b/newsdaemon.py index e47725b7e..4d3209f6f 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -882,7 +882,7 @@ def run_newswire_daemon(base_dir: str, httpd, def run_newswire_watchdog(project_version: str, httpd) -> None: """This tries to keep the newswire update thread running even if it dies """ - print('Starting newswire watchdog') + print('THREAD: Starting newswire watchdog') newswire_original = \ httpd.thrPostSchedule.clone(run_newswire_daemon) httpd.thrNewswireDaemon.start() @@ -891,6 +891,7 @@ def run_newswire_watchdog(project_version: str, httpd) -> None: if httpd.thrNewswireDaemon.is_alive(): continue httpd.thrNewswireDaemon.kill() + print('THREAD: restarting newswire watchdog') httpd.thrNewswireDaemon = \ newswire_original.clone(run_newswire_daemon) httpd.thrNewswireDaemon.start() diff --git a/outbox.py b/outbox.py index c8bd0f5a0..5e2943b8e 100644 --- a/outbox.py +++ b/outbox.py @@ -509,7 +509,7 @@ def post_message_to_outbox(session, translate: {}, followers_threads.pop(0) # create a thread to send the post to followers followers_thread = \ - send_to_followers_thread(server.session, + send_to_followers_thread(server, server.session, server.session_onion, server.session_i2p, base_dir, @@ -653,7 +653,7 @@ def post_message_to_outbox(session, translate: {}, print('c2s sender: ' + post_to_nickname + '@' + domain + ':' + str(port)) named_addresses_thread = \ - send_to_named_addresses_thread(server.session, + send_to_named_addresses_thread(server, server.session, server.session_onion, server.session_i2p, base_dir, post_to_nickname, @@ -668,6 +668,7 @@ def post_message_to_outbox(session, translate: {}, version, shared_items_federated_domains, shared_item_federation_tokens, - signing_priv_key_pem) + signing_priv_key_pem, + proxy_type) followers_threads.append(named_addresses_thread) return True diff --git a/person.py b/person.py index 15e11374e..69026c9fc 100644 --- a/person.py +++ b/person.py @@ -1687,17 +1687,13 @@ def valid_sending_actor(session, base_dir: str, if sending_actor.endswith(domain + '/users/' + nickname): return True - # get their actor - actor_json = \ - get_person_from_cache(base_dir, sending_actor, person_cache, True) - downloaded_actor = False - if not actor_json: - # download the actor - actor_json, _ = get_actor_json(domain, sending_actor, - True, False, debug, True, - signing_priv_key_pem, session) - if actor_json: - downloaded_actor = True + # download the actor + # NOTE: the actor should not be obtained from the local cache, + # because they may have changed fields which are being tested here, + # such as the bio length + actor_json, _ = get_actor_json(domain, sending_actor, + True, False, debug, True, + signing_priv_key_pem, session) if not actor_json: # if the actor couldn't be obtained then proceed anyway return True @@ -1776,9 +1772,8 @@ def valid_sending_actor(session, base_dir: str, sending_actor) return False - if downloaded_actor: - # if the actor is valid and was downloaded then - # store it in the cache, but don't write it to file - store_person_in_cache(base_dir, sending_actor, actor_json, - person_cache, False) + # if the actor is valid and was downloaded then + # store it in the cache, but don't write it to file + store_person_in_cache(base_dir, sending_actor, actor_json, + person_cache, False) return True diff --git a/posts.py b/posts.py index c5a15ea24..5171b6422 100644 --- a/posts.py +++ b/posts.py @@ -2494,6 +2494,7 @@ def send_post(signing_priv_key_pem: str, project_version: str, send_threads[0].kill() send_threads.pop(0) print('WARN: thread killed') + print('THREAD: thread_send_post') thr = \ thread_with_trace(target=thread_send_post, args=(session, @@ -2722,7 +2723,8 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, person_cache: {}, debug: bool, project_version: str, shared_items_token: str, group_account: bool, signing_priv_key_pem: str, - source_id: int) -> int: + source_id: int, curr_domain: str, + onion_domain: str, i2p_domain: str) -> int: """Sends a signed json object to an inbox/outbox """ if debug: @@ -2818,12 +2820,19 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, # shared_inbox is optional # get the senders private key + account_domain = origin_domain + if onion_domain: + if account_domain == onion_domain: + account_domain = curr_domain + if i2p_domain: + if account_domain == i2p_domain: + account_domain = curr_domain private_key_pem = \ - _get_person_key(nickname, domain, base_dir, 'private', debug) + _get_person_key(nickname, account_domain, base_dir, 'private', debug) if len(private_key_pem) == 0: if debug: print('DEBUG: Private key not found for ' + - nickname + '@' + domain + + nickname + '@' + account_domain + ' in ' + base_dir + '/keys/private') return 6 @@ -2849,6 +2858,20 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, # subsequent conversions after creating message body digest post_json_str = json.dumps(post_json_object) + # if the sender domain has changed from clearnet to onion or i2p + # then change the content of the post accordingly + if debug: + print('Checking for changed origin domain: ' + + domain + ' ' + curr_domain) + if domain != curr_domain: + if not curr_domain.endswith('.onion') and \ + not curr_domain.endswith('.i2p'): + if debug: + print('Changing post content sender domain from ' + + curr_domain + ' to ' + domain) + post_json_str = \ + post_json_str.replace(curr_domain, domain) + # construct the http header, including the message body digest signature_header_json = \ create_signed_header(None, private_key_pem, nickname, domain, port, @@ -2881,6 +2904,7 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, print('DEBUG: starting thread to send post') pprint(post_json_object) domain_full = get_full_domain(domain, port) + print('THREAD: thread_send_post 2') thr = \ thread_with_trace(target=thread_send_post, args=(session, @@ -2981,7 +3005,7 @@ def _is_profile_update(post_json_object: {}) -> bool: return False -def _send_to_named_addresses(session, session_onion, session_i2p, +def _send_to_named_addresses(server, session, session_onion, session_i2p, base_dir: str, nickname: str, domain: str, onion_domain: str, i2p_domain: str, port: int, @@ -2992,7 +3016,8 @@ def _send_to_named_addresses(session, session_onion, session_i2p, project_version: str, shared_items_federated_domains: [], shared_item_federation_tokens: {}, - signing_priv_key_pem: str) -> None: + signing_priv_key_pem: str, + proxy_type: str) -> None: """sends a post to the specific named addresses in to/cc """ if not session: @@ -3091,11 +3116,6 @@ def _send_to_named_addresses(session, session_onion, session_i2p, print('Not sending profile update to self. ' + nickname + '@' + domain_full) continue - if debug: - domain_full = get_full_domain(domain, port) - to_domain_full = get_full_domain(to_domain, to_port) - print('DEBUG: Post sending s2s: ' + nickname + '@' + domain_full + - ' to ' + to_nickname + '@' + to_domain_full) # if we have an alt onion domain and we are sending to # another onion domain then switch the clearnet @@ -3104,20 +3124,38 @@ def _send_to_named_addresses(session, session_onion, session_i2p, from_domain_full = get_full_domain(domain, port) from_http_prefix = http_prefix curr_session = session + curr_proxy_type = proxy_type + session_type = 'default' if onion_domain: - if to_domain.endswith('.onion'): + if not from_domain.endswith('.onion') and \ + to_domain.endswith('.onion'): from_domain = onion_domain from_domain_full = onion_domain from_http_prefix = 'http' curr_session = session_onion + port = 80 + to_port = 80 + curr_proxy_type = 'tor' + session_type = 'tor' if i2p_domain: - if to_domain.endswith('.i2p'): + if not from_domain.endswith('.i2p') and \ + to_domain.endswith('.i2p'): from_domain = i2p_domain from_domain_full = i2p_domain from_http_prefix = 'http' curr_session = session_i2p + port = 80 + to_port = 80 + curr_proxy_type = 'i2p' + session_type = 'i2p' cc_list = [] + if debug: + to_domain_full = get_full_domain(to_domain, to_port) + print('DEBUG: Post sending s2s: ' + + nickname + '@' + from_domain_full + + ' to ' + to_nickname + '@' + to_domain_full) + # if the "to" domain is within the shared items # federation list then send the token for this domain # so that it can request a catalog @@ -3129,6 +3167,16 @@ def _send_to_named_addresses(session, session_onion, session_i2p, group_account = has_group_type(base_dir, address, person_cache) + if not curr_session: + curr_session = create_session(curr_proxy_type) + if server: + if session_type == 'tor': + server.session_onion = curr_session + elif session_type == 'i2p': + server.session_i2p = curr_session + else: + server.session = curr_session + send_signed_json(post_json_object, curr_session, base_dir, nickname, from_domain, port, to_nickname, to_domain, to_port, @@ -3137,10 +3185,11 @@ def _send_to_named_addresses(session, session_onion, session_i2p, send_threads, post_log, cached_webfingers, person_cache, debug, project_version, shared_items_token, group_account, - signing_priv_key_pem, 34436782) + signing_priv_key_pem, 34436782, + domain, onion_domain, i2p_domain) -def send_to_named_addresses_thread(session, session_onion, session_i2p, +def send_to_named_addresses_thread(server, session, session_onion, session_i2p, base_dir: str, nickname: str, domain: str, onion_domain: str, i2p_domain: str, port: int, @@ -3151,12 +3200,14 @@ def send_to_named_addresses_thread(session, session_onion, session_i2p, project_version: str, shared_items_federated_domains: [], shared_item_federation_tokens: {}, - signing_priv_key_pem: str): + signing_priv_key_pem: str, + proxy_type: str): """Returns a thread used to send a post to named addresses """ + print('THREAD: _send_to_named_addresses') send_thread = \ thread_with_trace(target=_send_to_named_addresses, - args=(session, session_onion, session_i2p, + args=(server, session, session_onion, session_i2p, base_dir, nickname, domain, onion_domain, i2p_domain, port, http_prefix, federation_list, @@ -3166,7 +3217,8 @@ def send_to_named_addresses_thread(session, session_onion, session_i2p, project_version, shared_items_federated_domains, shared_item_federation_tokens, - signing_priv_key_pem), daemon=True) + signing_priv_key_pem, + proxy_type), daemon=True) try: send_thread.start() except SocketError as ex: @@ -3213,7 +3265,7 @@ def _sending_profile_update(post_json_object: {}) -> bool: return False -def send_to_followers(session, session_onion, session_i2p, +def send_to_followers(server, session, session_onion, session_i2p, base_dir: str, nickname: str, domain: str, onion_domain: str, i2p_domain: str, port: int, http_prefix: str, federation_list: [], @@ -3246,6 +3298,12 @@ def send_to_followers(session, session_onion, session_i2p, # this is after the message has arrived at the server client_to_server = False + curr_proxy_type = None + if domain.endswith('.onion'): + curr_proxy_type = 'tor' + elif domain.endswith('.i2p'): + curr_proxy_type = 'i2p' + # for each instance sending_start_time = datetime.datetime.utcnow() print('Sending post to followers begins ' + @@ -3289,10 +3347,6 @@ def send_to_followers(session, session_onion, session_i2p, if follower_domain.endswith('.i2p'): curr_session = session_i2p curr_http_prefix = 'http' - if not curr_session: - print('WARN: session not found when sending to follower ' + - follower_domain_url) - continue with_shared_inbox = \ _has_shared_inbox(curr_session, curr_http_prefix, follower_domain, @@ -3317,14 +3371,33 @@ def send_to_followers(session, session_onion, session_i2p, # have an alt onion domain then use the alt from_domain = domain from_http_prefix = http_prefix + session_type = 'default' if onion_domain: if to_domain.endswith('.onion'): from_domain = onion_domain from_http_prefix = 'http' + port = 80 + to_port = 80 + curr_proxy_type = 'tor' + session_type = 'tor' if i2p_domain: if to_domain.endswith('.i2p'): from_domain = i2p_domain from_http_prefix = 'http' + port = 80 + to_port = 80 + curr_proxy_type = 'i2p' + session_type = 'i2p' + + if not curr_session: + curr_session = create_session(curr_proxy_type) + if server: + if session_type == 'tor': + server.session_onion = curr_session + elif session_type == 'i2p': + server.session_i2p = curr_session + else: + server.session = curr_session if with_shared_inbox: to_nickname = follower_handles[index].split('@')[0] @@ -3357,7 +3430,8 @@ def send_to_followers(session, session_onion, session_i2p, send_threads, post_log, cached_webfingers, person_cache, debug, project_version, shared_items_token, group_account, - signing_priv_key_pem, 639342) + signing_priv_key_pem, 639342, + domain, onion_domain, i2p_domain) else: # send to individual followers without using a shared inbox for handle in follower_handles: @@ -3386,7 +3460,8 @@ def send_to_followers(session, session_onion, session_i2p, send_threads, post_log, cached_webfingers, person_cache, debug, project_version, shared_items_token, group_account, - signing_priv_key_pem, 634219) + signing_priv_key_pem, 634219, + domain, onion_domain, i2p_domain) time.sleep(4) @@ -3399,7 +3474,7 @@ def send_to_followers(session, session_onion, session_i2p, print('Sending post to followers ends ' + str(sending_mins) + ' mins') -def send_to_followers_thread(session, session_onion, session_i2p, +def send_to_followers_thread(server, session, session_onion, session_i2p, base_dir: str, nickname: str, domain: str, onion_domain: str, i2p_domain: str, port: int, http_prefix: str, federation_list: [], @@ -3412,9 +3487,10 @@ def send_to_followers_thread(session, session_onion, session_i2p, signing_priv_key_pem: str): """Returns a thread used to send a post to followers """ + print('THREAD: send_to_followers') send_thread = \ thread_with_trace(target=send_to_followers, - args=(session, session_onion, session_i2p, + args=(server, session, session_onion, session_i2p, base_dir, nickname, domain, onion_domain, i2p_domain, port, http_prefix, federation_list, diff --git a/reaction.py b/reaction.py index 33c5a0a43..e2804c61f 100644 --- a/reaction.py +++ b/reaction.py @@ -70,7 +70,9 @@ def _reactionpost(recent_posts_cache: {}, send_threads: [], post_log: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, + onion_domain: str, i2p_domain: str) -> {}: """Creates an emoji reaction actor is the person doing the reacting 'to' might be a specific person (actor) whose post was reaction @@ -141,7 +143,8 @@ def _reactionpost(recent_posts_cache: {}, send_threads, post_log, cached_webfingers, person_cache, debug, project_version, None, group_account, - signing_priv_key_pem, 7165392) + signing_priv_key_pem, 7165392, + curr_domain, onion_domain, i2p_domain) return new_reaction_json @@ -156,7 +159,8 @@ def reaction_post(recent_posts_cache: {}, send_threads: [], post_log: [], person_cache: {}, cached_webfingers: {}, debug: bool, project_version: str, - signing_priv_key_pem: str) -> {}: + signing_priv_key_pem: str, + curr_domain: str, onion_domain: str, i2p_domain: str) -> {}: """Adds a reaction to a given status post. This is only used by unit tests """ reaction_domain = get_full_domain(reaction_domain, reaction_port) @@ -172,7 +176,8 @@ def reaction_post(recent_posts_cache: {}, actor_reaction, client_to_server, send_threads, post_log, person_cache, cached_webfingers, - debug, project_version, signing_priv_key_pem) + debug, project_version, signing_priv_key_pem, + curr_domain, onion_domain, i2p_domain) def send_reaction_via_server(base_dir: str, session, diff --git a/schedule.py b/schedule.py index 46e9b1a19..025a0f02e 100644 --- a/schedule.py +++ b/schedule.py @@ -193,7 +193,7 @@ def run_post_schedule(base_dir: str, httpd, max_scheduled_posts: int): def run_post_schedule_watchdog(project_version: str, httpd) -> None: """This tries to keep the scheduled post thread running even if it dies """ - print('Starting scheduled post watchdog') + print('THREAD: Starting scheduled post watchdog') post_schedule_original = \ httpd.thrPostSchedule.clone(run_post_schedule) httpd.thrPostSchedule.start() @@ -202,6 +202,7 @@ def run_post_schedule_watchdog(project_version: str, httpd) -> None: if httpd.thrPostSchedule.is_alive(): continue httpd.thrPostSchedule.kill() + print('THREAD: restarting scheduled post watchdog') httpd.thrPostSchedule = \ post_schedule_original.clone(run_post_schedule) httpd.thrPostSchedule.start() diff --git a/scripts/missing_arguments b/scripts/missing_arguments new file mode 100755 index 000000000..7dfc4aec4 --- /dev/null +++ b/scripts/missing_arguments @@ -0,0 +1,7 @@ +#!/bin/bash +journalctl -u epicyon | grep 'required positional arguments' > .missing_arguments.txt +if [ ! -f .missing_arguments.txt ]; then + echo 'No missing arguments' +else + cat .missing_arguments.txt +fi diff --git a/scripts/threads b/scripts/threads new file mode 100755 index 000000000..3ef9c00e4 --- /dev/null +++ b/scripts/threads @@ -0,0 +1,7 @@ +#!/bin/bash +journalctl -u epicyon | grep 'THREAD:' > .threads.txt +if [ ! -f .threads.txt ]; then + echo 'No thread events' +else + cat .threads.txt +fi diff --git a/shares.py b/shares.py index bbca66365..af73be24d 100644 --- a/shares.py +++ b/shares.py @@ -1610,7 +1610,7 @@ def run_federated_shares_watchdog(project_version: str, httpd) -> None: """This tries to keep the federated shares update thread running even if it dies """ - print('Starting federated shares watchdog') + print('THREAD: Starting federated shares watchdog') federated_shares_original = \ httpd.thrPostSchedule.clone(run_federated_shares_daemon) httpd.thrFederatedSharesDaemon.start() @@ -1619,6 +1619,7 @@ def run_federated_shares_watchdog(project_version: str, httpd) -> None: if httpd.thrFederatedSharesDaemon.is_alive(): continue httpd.thrFederatedSharesDaemon.kill() + print('THREAD: restarting federated shares watchdog') httpd.thrFederatedSharesDaemon = \ federated_shares_original.clone(run_federated_shares_daemon) httpd.thrFederatedSharesDaemon.start() diff --git a/tests.py b/tests.py index d5bed8eb3..c852060ff 100644 --- a/tests.py +++ b/tests.py @@ -51,7 +51,7 @@ from posts import send_post_via_server from posts import seconds_between_published from follow import clear_follows from follow import clear_followers -from follow import send_follow_requestViaServer +from follow import send_follow_request_via_server from follow import send_unfollow_request_via_server from siteactive import site_is_active from utils import convert_published_to_local_timezone @@ -1393,7 +1393,8 @@ def test_post_message_between_servers(base_dir: str) -> None: 'alice', alice_domain, alice_port, [], status_number, False, bob_send_threads, bob_post_log, bob_person_cache, bob_cached_webfingers, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + bob_domain, None, None) for _ in range(20): if 'likes' in open(outbox_post_filename).read(): @@ -1415,7 +1416,8 @@ def test_post_message_between_servers(base_dir: str) -> None: status_number, '😀', False, bob_send_threads, bob_post_log, bob_person_cache, bob_cached_webfingers, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + bob_domain, None, None) for i in range(20): if 'reactions' in open(outbox_post_filename).read(): @@ -1451,10 +1453,11 @@ def test_post_message_between_servers(base_dir: str) -> None: object_url, False, bob_send_threads, bob_post_log, bob_person_cache, bob_cached_webfingers, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + bob_domain, None, None) announce_message_arrived = False outbox_message_arrived = False - for i in range(10): + for i in range(20): time.sleep(1) if not os.path.isdir(inbox_path): continue @@ -1597,7 +1600,8 @@ def test_follow_between_servers(base_dir: str) -> None: client_to_server, federation_list, alice_send_threads, alice_post_log, alice_cached_webfingers, alice_person_cache, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + alice_domain, None, None) print('send_result: ' + str(send_result)) for _ in range(16): @@ -1818,7 +1822,8 @@ def test_shared_items_federation(base_dir: str) -> None: client_to_server, federation_list, alice_send_threads, alice_post_log, alice_cached_webfingers, alice_person_cache, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + alice_domain, None, None) print('send_result: ' + str(send_result)) for _ in range(16): @@ -2270,7 +2275,8 @@ def test_group_follow(base_dir: str) -> None: client_to_server, federation_list, alice_send_threads, alice_post_log, alice_cached_webfingers, alice_person_cache, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + alice_domain, None, None) print('send_result: ' + str(send_result)) alice_following_filename = \ @@ -2348,7 +2354,8 @@ def test_group_follow(base_dir: str) -> None: client_to_server, federation_list, bob_send_threads, bob_post_log, bob_cached_webfingers, bob_person_cache, - True, __version__, signing_priv_key_pem) + True, __version__, signing_priv_key_pem, + bob_domain, None, None) print('send_result: ' + str(send_result)) bob_following_filename = \ @@ -3094,13 +3101,13 @@ def test_client_to_server(base_dir: str): print('\n\nAlice follows Bob') signing_priv_key_pem = None - send_follow_requestViaServer(alice_dir, session_alice, - 'alice', password, - alice_domain, alice_port, - 'bob', bob_domain, bob_port, - http_prefix, - cached_webfingers, person_cache, - True, __version__, signing_priv_key_pem) + send_follow_request_via_server(alice_dir, session_alice, + 'alice', password, + alice_domain, alice_port, + 'bob', bob_domain, bob_port, + http_prefix, + cached_webfingers, person_cache, + True, __version__, signing_priv_key_pem) alice_petnames_filename = alice_dir + '/accounts/' + \ 'alice@' + alice_domain + '/petnames.txt' alice_following_filename = \ @@ -3136,13 +3143,13 @@ def test_client_to_server(base_dir: str): alice_domain, alice_port) print('\n\nEVENT: Bob follows Alice') - send_follow_requestViaServer(alice_dir, session_alice, - 'bob', 'bobpass', - bob_domain, bob_port, - 'alice', alice_domain, alice_port, - http_prefix, - cached_webfingers, person_cache, - True, __version__, signing_priv_key_pem) + send_follow_request_via_server(alice_dir, session_alice, + 'bob', 'bobpass', + bob_domain, bob_port, + 'alice', alice_domain, alice_port, + http_prefix, + cached_webfingers, person_cache, + True, __version__, signing_priv_key_pem) for _ in range(10): if os.path.isfile(alice_dir + '/accounts/alice@' + alice_domain + '/followers.txt'): diff --git a/theme.py b/theme.py index f89d24d20..0d9fc584b 100644 --- a/theme.py +++ b/theme.py @@ -861,16 +861,11 @@ def set_theme(base_dir: str, name: str, domain: str, _remove_theme(base_dir) + # has the theme changed? themes = get_themes_list(base_dir) for theme_name in themes: theme_name_lower = theme_name.lower() if name == theme_name_lower: - allow_access = allow_local_network_access - try: - globals()['set_theme' + theme_name](base_dir, allow_access) - except BaseException: - print('EX: set_theme unable to set theme ' + theme_name) - if prev_theme_name: if prev_theme_name.lower() != theme_name_lower: # change the banner and profile image @@ -878,12 +873,14 @@ def set_theme(base_dir: str, name: str, domain: str, _set_theme_images(base_dir, name) _set_theme_fonts(base_dir, name) result = True + break if not result: # default _set_theme_default(base_dir, allow_local_network_access) result = True + # read theme settings from a json file in the theme directory variables_file = base_dir + '/theme/' + name + '/theme.json' if os.path.isfile(variables_file): _read_variables_file(base_dir, name, variables_file, diff --git a/threads.py b/threads.py index 0aca29b25..c17ad025e 100644 --- a/threads.py +++ b/threads.py @@ -75,6 +75,7 @@ class thread_with_trace(threading.Thread): def clone(self, func): """Create a clone """ + print('THREAD: clone') return thread_with_trace(target=func, args=self._args, daemon=True) diff --git a/webfinger.py b/webfinger.py index 3432af169..12a8e58a5 100644 --- a/webfinger.py +++ b/webfinger.py @@ -245,7 +245,7 @@ def webfinger_meta(http_prefix: str, domain_full: str) -> str: def webfinger_lookup(path: str, base_dir: str, - domain: str, onion_domain: str, + domain: str, onion_domain: str, i2p_domain: str, port: int, debug: bool) -> {}: """Lookup the webfinger endpoint for an account """ @@ -287,6 +287,11 @@ def webfinger_lookup(path: str, base_dir: str, if onion_domain in handle: handle = handle.replace(onion_domain, domain) onionify = True + i2pify = False + if i2p_domain: + if i2p_domain in handle: + handle = handle.replace(i2p_domain, domain) + i2pify = True # instance actor if handle.startswith('actor@'): handle = handle.replace('actor@', 'inbox@', 1) @@ -299,11 +304,14 @@ def webfinger_lookup(path: str, base_dir: str, if debug: print('DEBUG: WEBFINGER filename not found ' + filename) return None - if not onionify: + if not onionify and not i2pify: wf_json = load_json(filename) - else: + elif onionify: print('Webfinger request for onionified ' + handle) wf_json = load_json_onionify(filename, domain, onion_domain) + else: + print('Webfinger request for i2pified ' + handle) + wf_json = load_json_onionify(filename, domain, i2p_domain) if not wf_json: wf_json = {"nickname": "unknown"} return wf_json