indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Improve user agent blocking

merge-requests/30/head
Bob Mottram 2022-03-31 17:14:19 +01:00
parent c9cd6ea9ab
commit c395688c18
2 changed files with 5 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
blocking.py
View File

@ -327,10 +327,10 @@ def is_blocked_domain(base_dir: str, domain: str,
if not broch_mode_is_active(base_dir):
if blocked_cache:
for blocked_str in blocked_cache:
if '*@' + domain in blocked_str:
if blocked_str == '*@' + domain:
return True
if short_domain:
if '*@' + short_domain in blocked_str:
if blocked_str == '*@' + short_domain:
return True
else:
# instance block list
@ -339,10 +339,10 @@ def is_blocked_domain(base_dir: str, domain: str,
try:
with open(global_blocking_filename, 'r') as fp_blocked:
blocked_str = fp_blocked.read()
if '*@' + domain in blocked_str:
if '*@' + domain + '\n' in blocked_str:
return True
if short_domain:
if '*@' + short_domain in blocked_str:
if '*@' + short_domain + '\n' in blocked_str:
return True
except OSError as ex:
print('EX: unable to read ' + global_blocking_filename +

9
crawlers.py
View File

@ -113,14 +113,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
agent_str_lower = agent_str.lower()
for ua_block in default_user_agent_blocks:
if agent_str_lower.endswith(ua_block) or \
ua_block + "/" in agent_str_lower or \
ua_block + ")" in agent_str_lower or \
ua_block + ";" in agent_str_lower or \
ua_block + "," in agent_str_lower or \
ua_block + ">" in agent_str_lower or \
ua_block + "<" in agent_str_lower or \
ua_block + " " in agent_str_lower:
if ua_block in agent_str_lower:
print('Blocked User agent 1: ' + ua_block)
return True, blocked_cache_last_updated