indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Log sites which are unavailable

main
Bob Mottram 2023-09-15 22:04:31 +01:00
parent baf39c1518
commit eae05efd4c
15 changed files with 214 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
announce.py
View File

@ -149,7 +149,8 @@ def create_announce(session, base_dir: str, federation_list: [],
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str) -> {}:
onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Creates an announce message
Typically to_url will be https://www.w3.org/ns/activitystreams#Public
and cc_url might be a specific person favorited or repeated and the
@ -214,7 +215,7 @@ def create_announce(session, base_dir: str, federation_list: [],
debug, project_version, None, group_account,
signing_priv_key_pem, 639633,
curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
return new_announce
@ -227,7 +228,8 @@ def announce_public(session, base_dir: str, federation_list: [],
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str) -> {}:
onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Makes a public announcement
"""
from_domain = get_full_domain(domain, port)
@ -242,7 +244,8 @@ def announce_public(session, base_dir: str, federation_list: [],
person_cache, cached_webfingers,
debug, project_version,
signing_priv_key_pem, curr_domain,
onion_domain, i2p_domain)
onion_domain, i2p_domain,
sites_unavailable)
def send_announce_via_server(base_dir: str, session,

32
daemon.py
View File

@ -445,6 +445,7 @@ from fitnessFunctions import fitness_performance
from fitnessFunctions import fitness_thread
from fitnessFunctions import sorted_watch_points
from fitnessFunctions import html_watch_points_graph
from siteactive import load_unavailable_sites
from siteactive import referer_is_active
from webapp_likers import html_likers_of_post
from crawlers import update_known_crawlers
@ -1403,7 +1404,8 @@ class PubServer(BaseHTTPRequestHandler):
return True
if not referer_is_active(http_prefix,
referer_domain, ua_str,
calling_site_timeout):
calling_site_timeout,
self.server.sites_unavailable):
print('mastodon api referer url is not active ' +
referer_domain)
self._400()
@ -1596,7 +1598,8 @@ class PubServer(BaseHTTPRequestHandler):
if not referer_is_active(http_prefix,
referer_domain, ua_str,
calling_site_timeout):
calling_site_timeout,
self.server.sites_unavailable):
print('nodeinfo referer url is not active ' +
referer_domain)
self._400()
@ -1704,7 +1707,8 @@ class PubServer(BaseHTTPRequestHandler):
if not referer_is_active(http_prefix,
referer_domain, ua_str,
calling_site_timeout):
calling_site_timeout,
self.server.sites_unavailable):
print('security.txt referer url is not active ' +
referer_domain)
self._400()
@ -1911,7 +1915,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.content_license_url,
self.server.dogwhistles,
self.server.min_images_for_accounts,
self.server.buy_sites)
self.server.buy_sites,
self.server.sites_unavailable)
def _get_outbox_thread_index(self, nickname: str,
max_outbox_threads_per_account: int) -> int:
@ -4238,7 +4243,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.signing_priv_key_pem,
self.server.domain,
self.server.onion_domain,
self.server.i2p_domain)
self.server.i2p_domain,
self.server.sites_unavailable)
if '&submitUnblock=' in follow_confirm_params:
blocking_actor = \
@ -9809,7 +9815,7 @@ class PubServer(BaseHTTPRequestHandler):
getreq_start_time,
repeat_private: bool,
debug: bool,
curr_session) -> None:
curr_session, sites_unavailable: []) -> None:
"""The announce/repeat button was pressed on a post
"""
page_number = 1
@ -9899,7 +9905,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.signing_priv_key_pem,
self.server.domain,
onion_domain,
i2p_domain)
i2p_domain, sites_unavailable)
announce_filename = None
if announce_json:
# save the announce straight to the outbox
@ -10194,7 +10200,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.project_version,
signing_priv_key_pem,
proxy_type,
followers_sync_cache)
followers_sync_cache,
self.server.sites_unavailable)
origin_path_str_absolute = \
http_prefix + '://' + domain_full + origin_path_str
if calling_domain.endswith('.onion') and onion_domain:
@ -10355,7 +10362,8 @@ class PubServer(BaseHTTPRequestHandler):
debug,
self.server.project_version,
self.server.signing_priv_key_pem,
self.server.followers_sync_cache)
self.server.followers_sync_cache,
self.server.sites_unavailable)
origin_path_str_absolute = \
http_prefix + '://' + domain_full + origin_path_str
if calling_domain.endswith('.onion') and onion_domain:
@ -19852,7 +19860,8 @@ class PubServer(BaseHTTPRequestHandler):
getreq_start_time,
repeat_private,
self.server.debug,
curr_session)
curr_session,
self.server.sites_unavailable)
self.server.getreq_busy = False
return
@ -23933,6 +23942,9 @@ def run_daemon(max_shares_on_profile: int,
httpd.starting_daemon = True
# list of websites which are currently down
httpd.sites_unavailable = load_unavailable_sites(base_dir)
# maximum number of shared items attached to actors, as in
# https://codeberg.org/fediverse/fep/src/branch/main/fep/0837/fep-0837.md
httpd.max_shares_on_profile = max_shares_on_profile

8
epicyon.py
View File

@ -1596,6 +1596,7 @@ def _command_options() -> None:
if i2p_domain:
session_i2p = create_session('i2p')
followers_sync_cache = {}
sites_unavailable = []
manual_approve_follow_request(session, session_onion, session_i2p,
onion_domain, i2p_domain,
base_dir, http_prefix,
@ -1606,7 +1607,8 @@ def _command_options() -> None:
cached_webfingers, person_cache,
debug, __version__,
signing_priv_key_pem, proxy_type,
followers_sync_cache)
followers_sync_cache,
sites_unavailable)
sys.exit()
if argb.deny:
@ -1639,6 +1641,7 @@ def _command_options() -> None:
if i2p_domain:
session_i2p = create_session('i2p')
followers_sync_cache = {}
sites_unavailable = []
manual_deny_follow_request(session, session_onion, session_i2p,
onion_domain, i2p_domain,
base_dir, http_prefix,
@ -1649,7 +1652,8 @@ def _command_options() -> None:
cached_webfingers, person_cache,
debug, __version__,
signing_priv_key_pem,
followers_sync_cache)
followers_sync_cache,
sites_unavailable)
sys.exit()
if argb.followerspending:

15
follow.py
View File

@ -763,7 +763,8 @@ def followed_account_accepts(session, base_dir: str, http_prefix: str,
signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str,
followers_sync_cache: {}):
followers_sync_cache: {},
sites_unavailable: []):
"""The person receiving a follow request accepts the new follower
and sends back an Accept activity
"""
@ -818,7 +819,7 @@ def followed_account_accepts(session, base_dir: str, http_prefix: str,
person_cache, debug, project_version, None,
group_account, signing_priv_key_pem,
7856837, curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
def followed_account_rejects(session, session_onion, session_i2p,
@ -832,7 +833,8 @@ def followed_account_rejects(session, session_onion, session_i2p,
cached_webfingers: {}, person_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
followers_sync_cache: {}):
followers_sync_cache: {},
sites_unavailable: []):
"""The person receiving a follow request rejects the new follower
and sends back a Reject activity
"""
@ -901,7 +903,7 @@ def followed_account_rejects(session, session_onion, session_i2p,
group_account, signing_priv_key_pem,
6393063,
domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
def send_follow_request(session, base_dir: str,
@ -916,7 +918,8 @@ def send_follow_request(session, base_dir: str,
person_cache: {}, debug: bool,
project_version: str, signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str) -> {}:
onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Gets the json object for sending a follow request
"""
if not signing_priv_key_pem:
@ -1000,7 +1003,7 @@ def send_follow_request(session, base_dir: str,
debug, project_version, None, group_account,
signing_priv_key_pem, 8234389,
curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
return new_follow_json

3
importFollowing.py
View File

@ -170,7 +170,8 @@ def _update_import_following(base_dir: str,
httpd.signing_priv_key_pem,
httpd.domain,
httpd.onion_domain,
httpd.i2p_domain)
httpd.i2p_domain,
httpd.sites_unavailable)
# remove the followed handle from the import list
following_str = following_str.replace(orig_line + '\n', '')

49
inbox.py
View File

@ -3709,7 +3709,8 @@ def _send_to_group_members(server, session, session_onion, session_i2p,
person_cache: {}, debug: bool,
curr_domain: str,
onion_domain: str, i2p_domain: str,
signing_priv_key_pem: str) -> None:
signing_priv_key_pem: str,
sites_unavailable: []) -> None:
"""When a post arrives for a group send it out to the group members
"""
if debug:
@ -3761,7 +3762,8 @@ def _send_to_group_members(server, session, session_onion, session_i2p,
send_threads, post_log,
person_cache, cached_webfingers,
debug, __version__, signing_priv_key_pem,
curr_domain, onion_domain, i2p_domain)
curr_domain, onion_domain, i2p_domain,
sites_unavailable)
send_to_followers_thread(server, session, session_onion, session_i2p,
base_dir, nickname, domain,
@ -3772,7 +3774,8 @@ def _send_to_group_members(server, session, session_onion, session_i2p,
announce_json, debug, __version__,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem)
signing_priv_key_pem,
sites_unavailable)
def _inbox_update_calendar(base_dir: str, handle: str,
@ -3906,7 +3909,8 @@ def _bounce_dm(sender_post_id: str, session, http_prefix: str,
dm_license_url: str,
languages_understood: [],
bounce_is_chat: bool,
curr_domain: str, onion_domain: str, i2p_domain: str) -> bool:
curr_domain: str, onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> bool:
"""Sends a bounce message back to the sending handle
if a DM has been rejected
"""
@ -3986,7 +3990,7 @@ def _bounce_dm(sender_post_id: str, session, http_prefix: str,
person_cache, debug, __version__, None, group_account,
signing_priv_key_pem, 7238634,
curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
return True
@ -4003,7 +4007,8 @@ def _is_valid_dm(base_dir: str, nickname: str, domain: str, port: int,
signing_priv_key_pem: str,
dm_license_url: str,
languages_understood: [],
curr_domain: str, onion_domain: str, i2p_domain: str) -> bool:
curr_domain: str, onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> bool:
"""Is the given message a valid DM?
"""
if nickname == 'inbox':
@ -4107,7 +4112,8 @@ def _is_valid_dm(base_dir: str, nickname: str, domain: str, port: int,
languages_understood,
bounce_chat,
curr_domain,
onion_domain, i2p_domain)
onion_domain, i2p_domain,
sites_unavailable)
return False
# dm index will be updated
@ -4136,7 +4142,8 @@ def _receive_question_vote(server, base_dir: str, nickname: str, domain: str,
cw_lists: {}, lists_enabled: bool,
bold_reading: bool, dogwhistles: {},
min_images_for_accounts: [],
buy_sites: {}) -> None:
buy_sites: {},
sites_unavailable: []) -> None:
"""Updates the votes on a Question/poll
"""
# if this is a reply to a question then update the votes
@ -4219,7 +4226,8 @@ def _receive_question_vote(server, base_dir: str, nickname: str, domain: str,
post_json_object, debug, __version__,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem)
signing_priv_key_pem,
sites_unavailable)
def _create_reply_notification_file(base_dir: str, nickname: str, domain: str,
@ -4374,7 +4382,8 @@ def _inbox_after_initial(server, inbox_start_time,
languages_understood: [],
mitm: bool, bold_reading: bool,
dogwhistles: {},
max_hashtags: int, buy_sites: {}) -> bool:
max_hashtags: int, buy_sites: {},
sites_unavailable: []) -> bool:
""" Anything which needs to be done after initial checks have passed
"""
# if this is a clearnet instance then replace any onion/i2p
@ -4709,7 +4718,8 @@ def _inbox_after_initial(server, inbox_start_time,
cw_lists, lists_enabled,
bold_reading, dogwhistles,
server.min_images_for_accounts,
server.buy_sites)
server.buy_sites,
server.sites_unavailable)
fitness_performance(inbox_start_time, server.fitness,
'INBOX', '_receive_question_vote',
debug)
@ -4805,7 +4815,8 @@ def _inbox_after_initial(server, inbox_start_time,
dm_license_url,
languages_understood,
domain,
onion_domain, i2p_domain):
onion_domain, i2p_domain,
server.sites_unavailable):
if debug:
print('Invalid DM ' + str(post_json_object))
return False
@ -5091,7 +5102,8 @@ def _inbox_after_initial(server, inbox_start_time,
post_log, cached_webfingers,
person_cache, debug,
domain, onion_domain, i2p_domain,
signing_priv_key_pem)
signing_priv_key_pem,
sites_unavailable)
fitness_performance(inbox_start_time,
server.fitness,
'INBOX', '_send_to_group_members',
@ -5353,7 +5365,8 @@ def _receive_follow_request(session, session_onion, session_i2p,
this_domain: str, onion_domain: str,
i2p_domain: str, signing_priv_key_pem: str,
unit_test: bool, system_language: str,
followers_sync_cache: {}) -> bool:
followers_sync_cache: {},
sites_unavailable: []) -> bool:
"""Receives a follow request within the POST section of HTTPServer
"""
if not message_json['type'].startswith('Follow'):
@ -5632,7 +5645,7 @@ def _receive_follow_request(session, session_onion, session_i2p,
debug, project_version, True,
signing_priv_key_pem,
this_domain, onion_domain, i2p_domain,
followers_sync_cache)
followers_sync_cache, sites_unavailable)
def run_inbox_queue(server,
@ -6072,7 +6085,8 @@ def run_inbox_queue(server,
onion_domain, i2p_domain,
signing_priv_key_pem, unit_test,
system_language,
server.followers_sync_cache):
server.followers_sync_cache,
server.sites_unavailable):
if os.path.isfile(queue_filename):
try:
os.remove(queue_filename)
@ -6260,7 +6274,8 @@ def run_inbox_queue(server,
dm_license_url,
languages_understood, mitm,
bold_reading, dogwhistles,
max_hashtags, server.buy_sites)
max_hashtags, server.buy_sites,
server.sites_unavailable)
fitness_performance(inbox_start_time, server.fitness,
'INBOX', 'handle_after_initial',
debug)

11
like.py
View File

@ -79,7 +79,8 @@ def _create_like(recent_posts_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str) -> {}:
onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Creates a like
actor is the person doing the liking
'to' might be a specific person (actor) whose post was liked
@ -147,7 +148,7 @@ def _create_like(recent_posts_cache: {},
debug, project_version, None, group_account,
signing_priv_key_pem, 7367374,
curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
return new_like_json
@ -162,7 +163,8 @@ def like_post(recent_posts_cache: {},
person_cache: {}, cached_webfingers: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str, onion_domain: str, i2p_domain: str) -> {}:
curr_domain: str, onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Likes a given status post. This is only used by unit tests
"""
like_domain = get_full_domain(like_domain, like_port)
@ -178,7 +180,8 @@ def like_post(recent_posts_cache: {},
send_threads, post_log, person_cache,
cached_webfingers,
debug, project_version, signing_priv_key_pem,
curr_domain, onion_domain, i2p_domain)
curr_domain, onion_domain, i2p_domain,
sites_unavailable)
def send_like_via_server(base_dir: str, session,

24
manualapprove.py
View File

@ -35,7 +35,8 @@ def manual_deny_follow_request(session, session_onion, session_i2p,
debug: bool,
project_version: str,
signing_priv_key_pem: str,
followers_sync_cache: {}) -> None:
followers_sync_cache: {},
sites_unavailable: []) -> None:
"""Manually deny a follow request
"""
accounts_dir = acct_dir(base_dir, nickname, domain)
@ -76,7 +77,8 @@ def manual_deny_follow_request(session, session_onion, session_i2p,
cached_webfingers, person_cache,
debug, project_version,
signing_priv_key_pem,
followers_sync_cache)
followers_sync_cache,
sites_unavailable)
print('Follow request from ' + deny_handle + ' was denied.')
@ -92,7 +94,8 @@ def manual_deny_follow_request_thread(session, session_onion, session_i2p,
debug: bool,
project_version: str,
signing_priv_key_pem: str,
followers_sync_cache: {}) -> None:
followers_sync_cache: {},
sites_unavailable: []) -> None:
"""Manually deny a follow request, within a thread so that the
user interface doesn't lag
"""
@ -110,7 +113,8 @@ def manual_deny_follow_request_thread(session, session_onion, session_i2p,
debug,
project_version,
signing_priv_key_pem,
followers_sync_cache), daemon=True)
followers_sync_cache,
sites_unavailable), daemon=True)
begin_thread(thr, 'manual_deny_follow_request_thread')
send_threads.append(thr)
@ -149,7 +153,8 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
project_version: str,
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {}) -> None:
followers_sync_cache: {},
sites_unavailable: []) -> None:
"""Manually approve a follow request
"""
handle = nickname + '@' + domain
@ -285,7 +290,8 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
domain,
onion_domain,
i2p_domain,
followers_sync_cache)
followers_sync_cache,
sites_unavailable)
update_approved_followers = True
else:
# this isn't the approved follow so it will remain
@ -358,7 +364,8 @@ def manual_approve_follow_request_thread(session, session_onion, session_i2p,
project_version: str,
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {}) -> None:
followers_sync_cache: {},
sites_unavailable: []) -> None:
"""Manually approve a follow request, in a thread so as not to cause
the UI to lag
"""
@ -377,6 +384,7 @@ def manual_approve_follow_request_thread(session, session_onion, session_i2p,
project_version,
signing_priv_key_pem,
proxy_type,
followers_sync_cache), daemon=True)
followers_sync_cache,
sites_unavailable), daemon=True)
begin_thread(thr, 'manual_approve_follow_request_thread')
send_threads.append(thr)

9
outbox.py
View File

@ -239,7 +239,8 @@ def post_message_to_outbox(session, translate: {},
content_license_url: str,
dogwhistles: {},
min_images_for_accounts: [],
buy_sites: {}) -> bool:
buy_sites: {},
sites_unavailable: []) -> bool:
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
@ -654,7 +655,8 @@ def post_message_to_outbox(session, translate: {},
version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem)
signing_priv_key_pem,
sites_unavailable)
followers_threads.append(followers_thread)
if debug:
@ -788,6 +790,7 @@ def post_message_to_outbox(session, translate: {},
shared_item_federation_tokens,
signing_priv_key_pem,
proxy_type,
server.followers_sync_cache)
server.followers_sync_cache,
server.sites_unavailable)
followers_threads.append(named_addresses_thread)
return True

30
posts.py
View File

@ -3090,7 +3090,7 @@ def send_signed_json(post_json_object: {}, session, base_dir: str,
signing_priv_key_pem: str,
source_id: int, curr_domain: str,
onion_domain: str, i2p_domain: str,
extra_headers: {}) -> int:
extra_headers: {}, sites_unavailable: []) -> int:
"""Sends a signed json object to an inbox/outbox
"""
if debug:
@ -3110,7 +3110,7 @@ def send_signed_json(post_json_object: {}, session, base_dir: str,
to_domain = get_full_domain(to_domain, to_port)
to_domain_url = http_prefix + '://' + to_domain
if not site_is_active(to_domain_url, 10):
if not site_is_active(to_domain_url, 10, sites_unavailable):
print('send_signed_json domain is inactive: ' + to_domain_url)
return 9
print('Domain is active: ' + to_domain_url)
@ -3401,7 +3401,8 @@ def _send_to_named_addresses(server, session, session_onion, session_i2p,
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {}) -> None:
followers_sync_cache: {},
sites_unavailable: []) -> None:
"""sends a post to the specific named addresses in to/cc
"""
if not session:
@ -3598,7 +3599,7 @@ def _send_to_named_addresses(server, session, session_onion, session_i2p,
shared_items_token, group_account,
signing_priv_key_pem, 34436782,
domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
@ -3614,7 +3615,8 @@ def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
shared_item_federation_tokens: {},
signing_priv_key_pem: str,
proxy_type: str,
followers_sync_cache: {}):
followers_sync_cache: {},
sites_unavailable: []):
"""Returns a thread used to send a post to named addresses
"""
print('THREAD: _send_to_named_addresses')
@ -3632,7 +3634,8 @@ def send_to_named_addresses_thread(server, session, session_onion, session_i2p,
shared_item_federation_tokens,
signing_priv_key_pem,
proxy_type,
followers_sync_cache), daemon=True)
followers_sync_cache,
sites_unavailable), daemon=True)
if not begin_thread(send_thread, 'send_to_named_addresses_thread'):
print('WARN: socket error while starting ' +
'thread to send to named addresses.')
@ -3684,7 +3687,8 @@ def send_to_followers(server, session, session_onion, session_i2p,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str) -> None:
signing_priv_key_pem: str,
sites_unavailable: []) -> None:
"""sends a post to the followers of the given nickname
"""
print('send_to_followers')
@ -3748,7 +3752,7 @@ def send_to_followers(server, session, session_onion, session_i2p,
# check that the follower's domain is active
follower_domain_url = http_prefix + '://' + follower_domain
if not site_is_active(follower_domain_url, 10):
if not site_is_active(follower_domain_url, 10, sites_unavailable):
print('Sending post to followers domain is inactive: ' +
follower_domain_url)
continue
@ -3856,7 +3860,7 @@ def send_to_followers(server, session, session_onion, session_i2p,
shared_items_token, group_account,
signing_priv_key_pem, 639342,
domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
else:
# randomize the order of handles, so that we are not
# favoring any particular account in terms of its delivery time
@ -3890,7 +3894,7 @@ def send_to_followers(server, session, session_onion, session_i2p,
shared_items_token, group_account,
signing_priv_key_pem, 634219,
domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
time.sleep(4)
@ -3913,7 +3917,8 @@ def send_to_followers_thread(server, session, session_onion, session_i2p,
project_version: str,
shared_items_federated_domains: [],
shared_item_federation_tokens: {},
signing_priv_key_pem: str):
signing_priv_key_pem: str,
sites_unavailable: []):
"""Returns a thread used to send a post to followers
"""
print('THREAD: send_to_followers')
@ -3929,7 +3934,8 @@ def send_to_followers_thread(server, session, session_onion, session_i2p,
project_version,
shared_items_federated_domains,
shared_item_federation_tokens,
signing_priv_key_pem), daemon=True)
signing_priv_key_pem,
sites_unavailable), daemon=True)
if not begin_thread(send_thread, 'send_to_followers_thread'):
print('WARN: error while starting ' +
'thread to send to followers.')

11
reaction.py
View File

@ -73,7 +73,8 @@ def _reactionpost(recent_posts_cache: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str,
onion_domain: str, i2p_domain: str) -> {}:
onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Creates an emoji reaction
actor is the person doing the reacting
'to' might be a specific person (actor) whose post was reaction
@ -148,7 +149,7 @@ def _reactionpost(recent_posts_cache: {},
debug, project_version, None, group_account,
signing_priv_key_pem, 7165392,
curr_domain, onion_domain, i2p_domain,
extra_headers)
extra_headers, sites_unavailable)
return new_reaction_json
@ -164,7 +165,8 @@ def reaction_post(recent_posts_cache: {},
person_cache: {}, cached_webfingers: {},
debug: bool, project_version: str,
signing_priv_key_pem: str,
curr_domain: str, onion_domain: str, i2p_domain: str) -> {}:
curr_domain: str, onion_domain: str, i2p_domain: str,
sites_unavailable: []) -> {}:
"""Adds a reaction to a given status post. This is only used by unit tests
"""
reaction_domain = get_full_domain(reaction_domain, reaction_port)
@ -181,7 +183,8 @@ def reaction_post(recent_posts_cache: {},
send_threads, post_log, person_cache,
cached_webfingers,
debug, project_version, signing_priv_key_pem,
curr_domain, onion_domain, i2p_domain)
curr_domain, onion_domain, i2p_domain,
sites_unavailable)
def send_reaction_via_server(base_dir: str, session,

11
schedule.py
View File

@ -20,6 +20,7 @@ from utils import remove_eol
from outbox import post_message_to_outbox
from session import create_session
from threads import begin_thread
from siteactive import save_unavailable_sites
def _update_post_schedule(base_dir: str, handle: str, httpd,
@ -148,7 +149,8 @@ def _update_post_schedule(base_dir: str, handle: str, httpd,
httpd.content_license_url,
httpd.dogwhistles,
httpd.min_images_for_accounts,
httpd.buy_sites):
httpd.buy_sites,
httpd.sites_unavailable):
index_lines.remove(line)
try:
os.remove(post_filename)
@ -205,8 +207,15 @@ def run_post_schedule_watchdog(project_version: str, httpd) -> None:
post_schedule_original = \
httpd.thrPostSchedule.clone(run_post_schedule)
begin_thread(httpd.thrPostSchedule, 'run_post_schedule_watchdog')
curr_sites_unavailable = httpd.sites_unavailable.copy()
while True:
time.sleep(20)
# save the list of unavailable sites
if str(curr_sites_unavailable) != httpd.sites_unavailable:
save_unavailable_sites(httpd.base_dir, httpd.sites_unavailable)
curr_sites_unavailable = httpd.sites_unavailable.copy()
if httpd.thrPostSchedule.is_alive():
continue
httpd.thrPostSchedule.kill()

9
shares.py
View File

@ -1658,7 +1658,8 @@ def _update_federated_shares_cache(session, shared_items_federated_domains: [],
http_prefix: str,
tokens_json: {}, debug: bool,
system_language: str,
shares_file_type: str) -> None:
shares_file_type: str,
sites_unavailable: []) -> None:
"""Updates the cache of federated shares for the instance.
This enables shared items to be available even when other instances
might not be online
@ -1687,7 +1688,8 @@ def _update_federated_shares_cache(session, shared_items_federated_domains: [],
if not tokens_json.get(federated_domain_full):
# token has been obtained for the other domain
continue
if not site_is_active(http_prefix + '://' + federated_domain_full, 10):
if not site_is_active(http_prefix + '://' + federated_domain_full, 10,
sites_unavailable):
continue
if shares_file_type == 'shares':
url = http_prefix + '://' + federated_domain_full + '/catalog'
@ -1827,6 +1829,7 @@ def run_federated_shares_daemon(base_dir: str, httpd, http_prefix: str,
min_days = 7
max_days = 14
_generate_next_shares_token_update(base_dir, min_days, max_days)
sites_unavailable = []
while True:
shared_items_federated_domains_str = \
get_config_param(base_dir, 'sharedItemsFederatedDomains')
@ -1866,7 +1869,7 @@ def run_federated_shares_daemon(base_dir: str, httpd, http_prefix: str,
shared_items_federated_domains,
base_dir, domain_full, http_prefix,
tokens_json, debug, system_language,
shares_file_type)
shares_file_type, sites_unavailable)
time.sleep(seconds_per_hour * 6)

47
siteactive.py
View File

@ -11,6 +11,7 @@ __module_group__ = "Core"
import http.client
from urllib.parse import urlparse
import ssl
from socket import gaierror
class Result:
@ -93,7 +94,8 @@ def _site_active_http_request(loc, timeout: int):
return result
def site_is_active(url: str, timeout: int) -> bool:
def site_is_active(url: str, timeout: int,
sites_unavailable: []) -> bool:
"""Returns true if the current url is resolvable.
This can be used to check that an instance is online before
trying to send posts to it.
@ -110,6 +112,9 @@ def site_is_active(url: str, timeout: int) -> bool:
loc = _site_active_parse_url(url)
result = Result(url=url)
url2 = url
if '://' in url:
url2 = url.split('://')[1]
try:
result = _site_active_http_request(loc, timeout)
@ -119,14 +124,21 @@ def site_is_active(url: str, timeout: int) -> bool:
return True
except gaierror as ex:
print('EX: site_is_active gaierror ' + url + ' ' + str(ex))
if url2 not in sites_unavailable:
sites_unavailable.append(url2)
except BaseException as ex:
print('EX: site_is_active ' + url + ' ' + str(ex))
if url2 in sites_unavailable:
sites_unavailable.remove(url2)
return False
def referer_is_active(http_prefix: str,
referer_domain: str, ua_str: str,
calling_site_timeout: int) -> bool:
calling_site_timeout: int,
sites_unavailable: []) -> bool:
"""Returns true if the given referer is an active website
"""
referer_url = http_prefix + '://' + referer_domain
@ -136,4 +148,33 @@ def referer_is_active(http_prefix: str,
for end_ch in ending_chars:
if end_ch in referer_url:
referer_url = referer_url.split(end_ch)[0]
return site_is_active(referer_url, calling_site_timeout)
return site_is_active(referer_url, calling_site_timeout,
sites_unavailable)
def save_unavailable_sites(base_dir: str, sites_unavailable: []) -> None:
"""Save a list of unavailable sites
"""
unavailable_sites_filename = base_dir + '/accounts/unavailable_sites.txt'
sites_unavailable.sort()
try:
with open(unavailable_sites_filename, 'w+',
encoding='utf-8') as fp_sites:
for site in sites_unavailable:
fp_sites.write(site + '\n')
except OSError:
print('EX: unable to save unavailable sites')
def load_unavailable_sites(base_dir: str) -> []:
"""load a list of unavailable sites
"""
unavailable_sites_filename = base_dir + '/accounts/unavailable_sites.txt'
sites_unavailable = []
try:
with open(unavailable_sites_filename, 'r',
encoding='utf-8') as fp_sites:
sites_unavailable = fp_sites.read().split('\n')
except OSError:
print('EX: unable to save unavailable sites')
return sites_unavailable

33
tests.py
View File

@ -1476,6 +1476,7 @@ def test_post_message_between_servers(base_dir: str) -> None:
bob_post_log = []
bob_person_cache = {}
bob_cached_webfingers = {}
sites_unavailable = []
status_number = None
outbox_post_filename = None
outbox_path = alice_dir + '/accounts/alice@' + alice_domain + '/outbox'
@ -1492,7 +1493,7 @@ def test_post_message_between_servers(base_dir: str) -> None:
status_number, False, bob_send_threads, bob_post_log,
bob_person_cache, bob_cached_webfingers,
True, __version__, signing_priv_key_pem,
bob_domain, None, None)
bob_domain, None, None, sites_unavailable)
for _ in range(20):
if text_in_file('likes', outbox_post_filename):
@ -1508,6 +1509,7 @@ def test_post_message_between_servers(base_dir: str) -> None:
print('\n\n*******************************************************')
print("Bob reacts to Alice's post")
sites_unavailable = []
assert reaction_post({}, session_bob, bob_dir, federation_list,
'bob', bob_domain, bob_port, http_prefix,
'alice', alice_domain, alice_port, [],
@ -1515,7 +1517,7 @@ def test_post_message_between_servers(base_dir: str) -> None:
False, bob_send_threads, bob_post_log,
bob_person_cache, bob_cached_webfingers,
True, __version__, signing_priv_key_pem,
bob_domain, None, None)
bob_domain, None, None, sites_unavailable)
for _ in range(20):
if text_in_file('reactions', outbox_post_filename):
@ -1546,13 +1548,14 @@ def test_post_message_between_servers(base_dir: str) -> None:
print('outbox items before announce: ' + str(outbox_before_announce_count))
assert outbox_before_announce_count == 0
assert before_announce_count == 0
sites_unavailable = []
announce_public(session_bob, bob_dir, federation_list,
'bob', bob_domain, bob_port, http_prefix,
object_url,
False, bob_send_threads, bob_post_log,
bob_person_cache, bob_cached_webfingers,
True, __version__, signing_priv_key_pem,
bob_domain, None, None)
bob_domain, None, None, sites_unavailable)
announce_message_arrived = False
outbox_message_arrived = False
for _ in range(20):
@ -1688,6 +1691,7 @@ def test_follow_between_servers(base_dir: str) -> None:
alice_person_cache = {}
alice_cached_webfingers = {}
alice_post_log = []
sites_unavailable = []
bob_actor = http_prefix + '://' + bob_address + '/users/bob'
signing_priv_key_pem = None
send_result = \
@ -1700,7 +1704,7 @@ def test_follow_between_servers(base_dir: str) -> None:
alice_send_threads, alice_post_log,
alice_cached_webfingers, alice_person_cache,
True, __version__, signing_priv_key_pem,
alice_domain, None, None)
alice_domain, None, None, sites_unavailable)
print('send_result: ' + str(send_result))
for _ in range(16):
@ -1919,6 +1923,7 @@ def test_shared_items_federation(base_dir: str) -> None:
alice_person_cache = {}
alice_cached_webfingers = {}
alice_post_log = []
sites_unavailable = []
bob_actor = http_prefix + '://' + bob_address + '/users/bob'
send_result = \
send_follow_request(session_alice, alice_dir,
@ -1930,7 +1935,7 @@ def test_shared_items_federation(base_dir: str) -> None:
alice_send_threads, alice_post_log,
alice_cached_webfingers, alice_person_cache,
True, __version__, signing_priv_key_pem,
alice_domain, None, None)
alice_domain, None, None, sites_unavailable)
print('send_result: ' + str(send_result))
for _ in range(16):
@ -2397,6 +2402,7 @@ def test_group_follow(base_dir: str) -> None:
alice_person_cache = {}
alice_cached_webfingers = {}
alice_post_log = []
sites_unavailable = []
# aliceActor = http_prefix + '://' + alice_address + '/users/alice'
testgroup_actor = \
http_prefix + '://' + testgroupAddress + '/users/testgroup'
@ -2411,7 +2417,7 @@ def test_group_follow(base_dir: str) -> None:
alice_send_threads, alice_post_log,
alice_cached_webfingers, alice_person_cache,
True, __version__, signing_priv_key_pem,
alice_domain, None, None)
alice_domain, None, None, sites_unavailable)
print('send_result: ' + str(send_result))
alice_following_filename = \
@ -2475,6 +2481,7 @@ def test_group_follow(base_dir: str) -> None:
bob_person_cache = {}
bob_cached_webfingers = {}
bob_post_log = []
sites_unavailable = []
# bob_actor = http_prefix + '://' + bob_address + '/users/bob'
testgroup_actor = \
http_prefix + '://' + testgroupAddress + '/users/testgroup'
@ -2489,7 +2496,7 @@ def test_group_follow(base_dir: str) -> None:
bob_send_threads, bob_post_log,
bob_cached_webfingers, bob_person_cache,
True, __version__, signing_priv_key_pem,
bob_domain, None, None)
bob_domain, None, None, sites_unavailable)
print('send_result: ' + str(send_result))
bob_following_filename = \
@ -4100,11 +4107,15 @@ def _test_jsonld():
def _test_site_active():
print('test_site_is_active')
timeout = 10
sites_unavailable = []
# at least one site should resolve
if not site_is_active('https://archive.org', timeout):
if not site_is_active('https://wikipedia.org', timeout):
assert site_is_active('https://mastodon.social', timeout)
assert not site_is_active('https://notarealwebsite.a.b.c', timeout)
if not site_is_active('https://archive.org', timeout, sites_unavailable):
if not site_is_active('https://wikipedia.org', timeout,
sites_unavailable):
assert site_is_active('https://mastodon.social', timeout,
sites_unavailable)
assert not site_is_active('https://notarealwebsite.a.b.c', timeout,
sites_unavailable)
def _test_strip_html():