indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Block some ai crawlers

merge-requests/30/head
Bob Mottram 2024-01-12 12:03:44 +00:00
parent abf2d925e8
commit 62b893dd8d
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crawlers.py
View File

@ -125,7 +125,9 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
if agent_str: if agent_str:
# is this a web crawler? If so then block it by default # is this a web crawler? If so then block it by default
# unless this is a news instance or if it is in the allowed list # unless this is a news instance or if it is in the allowed list
bot_strings = ('bot/', 'bot-', '/bot', '/robot', 'gptbot') bot_strings = ('bot/', 'bot-', '/bot', '/robot', 'gptbot',
'-ai/', ' ai/', '-ai ', ' ai ', 'spider/',
'externalhit/')
contains_bot_string = False contains_bot_string = False
for bot_str in bot_strings: for bot_str in bot_strings:
if bot_str in agent_str_lower: if bot_str in agent_str_lower: