indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Improve blocking of user agent domains

main
Bob Mottram 2022-03-31 16:12:30 +01:00
parent 592fb92539
commit 92b9942183
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crawlers.py
View File

@ -113,8 +113,14 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
agent_str_lower = agent_str.lower()
for ua_block in default_user_agent_blocks:
if ua_block in agent_str_lower:
print('Blocked User agent: ' + ua_block)
if agent_str_lower.endswith(ua_block) or \
ua_block + "/" in agent_str_lower or \
ua_block + ")" in agent_str_lower or \
ua_block + ";" in agent_str_lower or \
ua_block + ">" in agent_str_lower or \
ua_block + "<" in agent_str_lower or \
ua_block + " " in agent_str_lower:
print('Blocked User agent 1: ' + ua_block)
return True, blocked_cache_last_updated
agent_domain = None
@ -176,5 +182,5 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
is_blocked_domain(base_dir, agent_domain, blocked_cache)
# if self.server.debug:
if blocked_ua:
print('Blocked User agent: ' + agent_domain)
print('Blocked User agent 2: ' + agent_domain)
return blocked_ua, blocked_cache_last_updated