indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More crawlers

main
Bob Mottram 2024-05-06 22:39:59 +01:00
parent 9057c542f1
commit 2ea94354da
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers.py
View File

@ -129,7 +129,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
bot_strings = ('bot/', 'bot-', '/bot', '/robot', 'gptbot',
'-ai/', ' ai/', '-ai ', ' ai ', 'spider/',
'externalhit/', 'chatgpt', 'google',
'anthropic')
'anthropic', 'facebook', 'slurp', 'crawler',
'crawling', 'gigablast', 'archive.org')
contains_bot_string = False
for bot_str in bot_strings:
if bot_str in agent_str_lower: