From 12b112ae9769d0e233fa80ffc7c169022b25537c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Apr 2022 18:24:56 +0100 Subject: [PATCH 1/3] Emoji debug --- content.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/content.py b/content.py index 1a711e21f..34dd1bd48 100644 --- a/content.py +++ b/content.py @@ -355,11 +355,12 @@ def replace_emoji_from_tags(session, base_dir: str, replace_char) replaced = True except BaseException: - print('EX: replace_emoji_from_tags 1 ' + - 'no conversion of ' + - str(icon_name) + ' to chr ' + - tag_item['name'] + ' ' + - tag_item['icon']['url']) + if debug: + print('EX: replace_emoji_from_tags 1 ' + + 'no conversion of ' + + str(icon_name) + ' to chr ' + + tag_item['name'] + ' ' + + tag_item['icon']['url']) if not replaced: _save_custom_emoji(session, base_dir, tag_item['name'], @@ -377,11 +378,13 @@ def replace_emoji_from_tags(session, base_dir: str, replaced = True except BaseException: icon_code_sequence = '' - print('EX: replace_emoji_from_tags 2 ' + - 'no conversion of ' + - str(icode) + ' to chr ' + - tag_item['name'] + ' ' + - tag_item['icon']['url']) + if debug: + print('EX: ' + + 'replace_emoji_from_tags 2 ' + + 'no conversion of ' + + str(icode) + ' to chr ' + + tag_item['name'] + ' ' + + tag_item['icon']['url']) if not replaced: _save_custom_emoji(session, base_dir, tag_item['name'], From 0fddc2204d856dd21d143fa811e71a286e1ad72e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Apr 2022 13:17:00 +0100 Subject: [PATCH 2/3] Static analysis for thread function arguments --- daemon.py | 7 +++- posts.py | 17 +++++--- tests.py | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/daemon.py b/daemon.py index ecdfc452f..a50bce011 100644 --- a/daemon.py +++ b/daemon.py @@ -1633,9 +1633,10 @@ class PubServer(BaseHTTPRequestHandler): account_outbox_thread_name + '/' + str(self.server.outbox_thread_index[account_outbox_thread_name])) print('THREAD: _post_to_outbox') + message_json_copy = message_json.copy() self.server.outboxThread[account_outbox_thread_name][index] = \ thread_with_trace(target=self._post_to_outbox, - args=(message_json.copy(), + args=(message_json_copy, self.server.project_version, None, curr_session, proxy_type), daemon=True) @@ -21111,12 +21112,15 @@ def run_daemon(check_actor_timeout: int, args=(base_dir, httpd.fitness), daemon=True) httpd.thrFitness.start() + httpd.recent_posts_cache = {} + print('THREAD: Creating cache expiry thread') httpd.thrCache = \ thread_with_trace(target=expire_cache, args=(base_dir, httpd.person_cache, httpd.http_prefix, archive_dir, + httpd.recent_posts_cache, httpd.maxPostsInBox), daemon=True) httpd.thrCache.start() @@ -21150,7 +21154,6 @@ def run_daemon(check_actor_timeout: int, else: httpd.thrSharesExpire.start() - httpd.recent_posts_cache = {} httpd.max_recent_posts = max_recent_posts httpd.iconsCache = {} httpd.fontsCache = {} diff --git a/posts.py b/posts.py index d53778500..30591ebf9 100644 --- a/posts.py +++ b/posts.py @@ -2495,14 +2495,16 @@ def send_post(signing_priv_key_pem: str, project_version: str, send_threads.pop(0) print('WARN: thread killed') print('THREAD: thread_send_post') + signature_header_json_copy = signature_header_json.copy() + signature_header_json_ld_copy = signature_header_json_ld.copy() thr = \ thread_with_trace(target=thread_send_post, args=(session, post_json_str, federation_list, inbox_url, base_dir, - signature_header_json.copy(), - signature_header_json_ld.copy(), + signature_header_json_copy, + signature_header_json_ld_copy, post_log, debug, http_prefix, domain_full), daemon=True) send_threads.append(thr) @@ -2905,14 +2907,16 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, pprint(post_json_object) domain_full = get_full_domain(domain, port) print('THREAD: thread_send_post 2') + signature_header_json_copy = signature_header_json.copy(), + signature_header_json_ld_copy = signature_header_json_ld.copy(), thr = \ thread_with_trace(target=thread_send_post, args=(session, post_json_str, federation_list, inbox_url, base_dir, - signature_header_json.copy(), - signature_header_json_ld.copy(), + signature_header_json_copy, + signature_header_json_ld_copy, post_log, debug, http_prefix, domain_full), daemon=True) send_threads.append(thr) @@ -3488,6 +3492,7 @@ def send_to_followers_thread(server, session, session_onion, session_i2p, """Returns a thread used to send a post to followers """ print('THREAD: send_to_followers') + post_json_object_copy = post_json_object.copy() send_thread = \ thread_with_trace(target=send_to_followers, args=(server, session, session_onion, session_i2p, @@ -3496,7 +3501,7 @@ def send_to_followers_thread(server, session, session_onion, session_i2p, http_prefix, federation_list, send_threads, post_log, cached_webfingers, person_cache, - post_json_object.copy(), debug, + post_json_object_copy, debug, project_version, shared_items_federated_domains, shared_item_federation_tokens, @@ -4118,7 +4123,7 @@ def _create_box_indexed(recent_posts_cache: {}, def expire_cache(base_dir: str, person_cache: {}, http_prefix: str, archive_dir: str, recent_posts_cache: {}, - max_posts_in_box=32000): + max_posts_in_box: int): """Thread used to expire actors from the cache and archive old posts """ while True: diff --git a/tests.py b/tests.py index 3b3490c75..3ac6ca736 100644 --- a/tests.py +++ b/tests.py @@ -678,7 +678,7 @@ def _test_cache(): assert result['test'] == 'This is a test' -def _test_threads_function(param: str): +def _test_threads_function(param1: str, param2: str): for _ in range(10000): time.sleep(2) @@ -687,7 +687,7 @@ def _test_threads(): print('test_threads') thr = \ thread_with_trace(target=_test_threads_function, - args=('test',), + args=('test', 'test2'), daemon=True) thr.start() assert thr.is_alive() is True @@ -4869,6 +4869,122 @@ def _test_post_field_names(source_file: str, fieldnames: []): assert False +def _test_thread_functions(): + print('test_thread_functions') + modules = {} + threads_called_in_modules = [] + + # get the source for each module + for _, _, files in os.walk('.'): + for source_file in files: + if not source_file.endswith('.py'): + continue + if source_file.startswith('.#'): + continue + if source_file.startswith('flycheck_'): + continue + mod_name = source_file.replace('.py', '') + if mod_name == 'threads': + # don't test the threads module itself + continue + modules[mod_name] = { + 'functions': [] + } + source_str = '' + with open(source_file, 'r') as fp_src: + source_str = fp_src.read() + modules[mod_name]['source'] = source_str + if 'thread_with_trace(' in source_str: + threads_called_in_modules.append(mod_name) + with open(source_file, 'r') as fp_src: + lines = fp_src.readlines() + modules[mod_name]['lines'] = lines + + for mod_name in threads_called_in_modules: + thread_sections = \ + modules[mod_name]['source'].split('thread_with_trace(') + ctr = 0 + for thread_str in thread_sections: + if ctr == 0 or ',' not in thread_str: + ctr += 1 + continue + thread_function_args = thread_str.split(',') + first_parameter = thread_function_args[0] + if 'target=' not in first_parameter: + ctr += 1 + continue + thread_function_name = first_parameter.split('target=')[1].strip() + if not thread_function_name: + ctr += 1 + continue + if thread_function_name.startswith('self.'): + thread_function_name = thread_function_name.split('self.')[1] + # is this function declared at the top of the module + # or defined within the module? + import_str = ' import ' + thread_function_name + def_str = 'def ' + thread_function_name + '(' + if import_str not in modules[mod_name]['source']: + if def_str not in modules[mod_name]['source']: + print(mod_name + ' ' + first_parameter) + print(import_str + ' not in ' + mod_name) + assert(False) + if def_str in modules[mod_name]['source']: + defininition_module = mod_name + else: + # which module is the thread function defined within? + test_str = modules[mod_name]['source'].split(import_str)[0] + defininition_module = test_str.split('from ')[-1] + print('Thread function ' + thread_function_name + + ' defined in ' + defininition_module) + # check the function arguments + second_parameter = thread_function_args[1] + if 'args=' not in second_parameter: + print('No args parameter in ' + thread_function_name + + ' module ' + mod_name) + ctr += 1 + continue + arg_ctr = 0 + calling_function_args_list = [] + for func_arg in thread_function_args: + if arg_ctr == 0: + arg_ctr += 1 + continue + last_arg = False + if '(' in func_arg: + func_arg = func_arg.split('(')[1] + + if func_arg.endswith(')'): + func_arg = func_arg.split(')')[0] + last_arg = True + func_arg = func_arg.strip() + calling_function_args_list.append(func_arg) + if last_arg: + break + arg_ctr += 1 +# print(mod_name + ' ' + thread_function_name + ' ' + +# str(calling_function_args_list)) + # get the function definition arguments + test_str = \ + modules[defininition_module]['source'].split(def_str)[1] + test_str = test_str.split(')')[0] + definition_function_args_list = test_str.split(',') + if len(definition_function_args_list) > 0: + if definition_function_args_list[0] == 'self': + definition_function_args_list = \ + definition_function_args_list[1:] + if len(definition_function_args_list) != \ + len(calling_function_args_list): + print('Thread function ' + thread_function_name + + ' has ' + str(len(definition_function_args_list)) + + ' arguments, but ' + + str(len(calling_function_args_list)) + + ' were given in module ' + mod_name) + print(str(definition_function_args_list)) + print(str(calling_function_args_list)) + assert(False) + ctr += 1 + + def _test_functions(): print('test_functions') function = {} @@ -6779,6 +6895,7 @@ def run_all_tests(): ['queue_json', 'post_json_object', 'message_json', 'liked_post_json']) _test_checkbox_names() + _test_thread_functions() _test_functions() _test_bold_reading() _test_published_to_local_timezone() From e2bc95c9dbd87d47506413fe7c4671dbe553427a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Apr 2022 13:43:20 +0100 Subject: [PATCH 3/3] Object copy needs to be inline --- daemon.py | 3 +-- posts.py | 15 +++++---------- tests.py | 4 ++-- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/daemon.py b/daemon.py index a50bce011..87825fd28 100644 --- a/daemon.py +++ b/daemon.py @@ -1633,10 +1633,9 @@ class PubServer(BaseHTTPRequestHandler): account_outbox_thread_name + '/' + str(self.server.outbox_thread_index[account_outbox_thread_name])) print('THREAD: _post_to_outbox') - message_json_copy = message_json.copy() self.server.outboxThread[account_outbox_thread_name][index] = \ thread_with_trace(target=self._post_to_outbox, - args=(message_json_copy, + args=(message_json.copy(), self.server.project_version, None, curr_session, proxy_type), daemon=True) diff --git a/posts.py b/posts.py index 30591ebf9..09019cbbe 100644 --- a/posts.py +++ b/posts.py @@ -2495,16 +2495,14 @@ def send_post(signing_priv_key_pem: str, project_version: str, send_threads.pop(0) print('WARN: thread killed') print('THREAD: thread_send_post') - signature_header_json_copy = signature_header_json.copy() - signature_header_json_ld_copy = signature_header_json_ld.copy() thr = \ thread_with_trace(target=thread_send_post, args=(session, post_json_str, federation_list, inbox_url, base_dir, - signature_header_json_copy, - signature_header_json_ld_copy, + signature_header_json.copy(), + signature_header_json_ld.copy(), post_log, debug, http_prefix, domain_full), daemon=True) send_threads.append(thr) @@ -2907,16 +2905,14 @@ def send_signed_json(post_json_object: {}, session, base_dir: str, pprint(post_json_object) domain_full = get_full_domain(domain, port) print('THREAD: thread_send_post 2') - signature_header_json_copy = signature_header_json.copy(), - signature_header_json_ld_copy = signature_header_json_ld.copy(), thr = \ thread_with_trace(target=thread_send_post, args=(session, post_json_str, federation_list, inbox_url, base_dir, - signature_header_json_copy, - signature_header_json_ld_copy, + signature_header_json.copy(), + signature_header_json_ld.copy(), post_log, debug, http_prefix, domain_full), daemon=True) send_threads.append(thr) @@ -3492,7 +3488,6 @@ def send_to_followers_thread(server, session, session_onion, session_i2p, """Returns a thread used to send a post to followers """ print('THREAD: send_to_followers') - post_json_object_copy = post_json_object.copy() send_thread = \ thread_with_trace(target=send_to_followers, args=(server, session, session_onion, session_i2p, @@ -3501,7 +3496,7 @@ def send_to_followers_thread(server, session, session_onion, session_i2p, http_prefix, federation_list, send_threads, post_log, cached_webfingers, person_cache, - post_json_object_copy, debug, + post_json_object.copy(), debug, project_version, shared_items_federated_domains, shared_item_federation_tokens, diff --git a/tests.py b/tests.py index 3ac6ca736..fd0fed64c 100644 --- a/tests.py +++ b/tests.py @@ -4950,10 +4950,10 @@ def _test_thread_functions(): arg_ctr += 1 continue last_arg = False - if '(' in func_arg: + if '(' in func_arg and '()' not in func_arg: func_arg = func_arg.split('(')[1] - if func_arg.endswith(')'): + if func_arg.endswith(')') and '()' not in func_arg: func_arg = func_arg.split(')')[0] last_arg = True func_arg = func_arg.strip()