indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Less indentation

main
Bob Mottram 2024-05-07 10:20:10 +01:00
parent 6a9aea9a3c
commit a215fe2e6a
1 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers.py
View File

@ -126,7 +126,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
if agent_str: if agent_str:
# is this a web crawler? If so then block it by default # is this a web crawler? If so then block it by default
# unless this is a news instance or if it is in the allowed list # unless this is a news instance or if it is in the allowed list
bot_strings = ('bot/', 'bot-', '/bot', '_bot', 'bot_', 'bot;', ' bot ', bot_strings = (
'bot/', 'bot-', '/bot', '_bot', 'bot_', 'bot;', ' bot ',
'/robot', 'gptbot', '-ai/', ' ai/', '-ai ', '/robot', 'gptbot', '-ai/', ' ai/', '-ai ',
' ai ', 'spider/', 'spider.ht', '/spider.', '-spider', ' ai ', 'spider/', 'spider.ht', '/spider.', '-spider',
'externalhit/', 'chatgpt', 'google', 'anthropic', 'externalhit/', 'chatgpt', 'google', 'anthropic',