indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Tidying

main
Bob Mottram 2024-12-15 18:53:17 +00:00
parent eaf57a9781
commit 3b55ef745e
1 changed files with 36 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
crawlers.py
View File

@ -229,80 +229,41 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
if blocked_ua: if blocked_ua:
print('BLOCK: Blocked User agent 2: ' + agent_domain) print('BLOCK: Blocked User agent 2: ' + agent_domain)
# optionally block military domains on a per account basis block_dicts = {
if not blocked_ua and block_military: "military": block_military,
if '/users/' in path: "government": block_government,
# which accounts is this? "bluesky": block_bluesky
nickname = path.split('/users/')[1] }
if '/' in nickname: for block_type, block_dict in block_dicts.items():
nickname = nickname.split('/')[0] if blocked_ua or not block_dict:
# does this account block military domains? continue
if block_military.get(nickname): if '/users/' not in path:
mil_domains = get_mil_domains_list() continue
for domain_str in mil_domains: # which accounts is this?
if '.' not in domain_str: nickname = path.split('/users/')[1]
tld = domain_str if '/' in nickname:
if agent_domain.endswith('.' + tld): nickname = nickname.split('/')[0]
blocked_ua = True # does this account block?
print('BLOCK: Blocked military tld user agent: ' + if not block_dict.get(nickname):
agent_domain) continue
break if block_type == "military":
else: blk_domains = get_mil_domains_list()
if agent_domain.endswith(domain_str): elif block_type == "government":
blocked_ua = True blk_domains = get_gov_domains_list()
print('BLOCK: Blocked military user agent: ' + else:
agent_domain) blk_domains = get_bsky_domains_list()
break for domain_str in blk_domains:
if '.' not in domain_str:
# optionally block government domains on a per account basis tld = domain_str
if not blocked_ua and block_government: if agent_domain.endswith('.' + tld):
if '/users/' in path: blocked_ua = True
# which accounts is this? print('BLOCK: Blocked ' + block_type +
nickname = path.split('/users/')[1] ' tld user agent: ' + agent_domain)
if '/' in nickname: break
nickname = nickname.split('/')[0] elif agent_domain.endswith(domain_str):
# does this account block government domains? blocked_ua = True
if block_government.get(nickname): print('BLOCK: Blocked ' + block_type +
gov_domains = get_gov_domains_list() ' user agent: ' + agent_domain)
for domain_str in gov_domains: break
if '.' not in domain_str:
tld = domain_str
if agent_domain.endswith('.' + tld):
blocked_ua = True
print('BLOCK: ' +
'Blocked government tld user agent: ' +
agent_domain)
break
else:
if agent_domain.endswith(domain_str):
blocked_ua = True
print('BLOCK: Blocked government user agent: ' +
agent_domain)
break
# optionally block bluesky bridges on a per account basis
if not blocked_ua and block_bluesky:
if '/users/' in path:
# which accounts is this?
nickname = path.split('/users/')[1]
if '/' in nickname:
nickname = nickname.split('/')[0]
# does this account block bluesky bridges?
if block_bluesky.get(nickname):
bsky_domains = get_bsky_domains_list()
for domain_str in bsky_domains:
if '.' not in domain_str:
tld = domain_str
if agent_domain.endswith('.' + tld):
blocked_ua = True
print('BLOCK: Blocked bluesky tld user agent: ' +
agent_domain)
break
else:
if agent_domain.endswith(domain_str):
blocked_ua = True
print('BLOCK: Blocked bluesky user agent: ' +
agent_domain)
break
return blocked_ua, blocked_cache_last_updated, False return blocked_ua, blocked_cache_last_updated, False