indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

duplicate

main
Bob Mottram 2024-05-06 23:24:48 +01:00
parent 18508fab89
commit 6a9aea9a3c
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawlers.py
View File

@ -131,7 +131,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
' ai ', 'spider/', 'spider.ht', '/spider.', '-spider',
'externalhit/', 'chatgpt', 'google', 'anthropic',
'facebook', 'slurp', 'crawler', 'crawling', ' crawl ',
'gigablast', 'archive.org', 'httrack', 'spider/',
'gigablast', 'archive.org', 'httrack',
'spider-', ' spider ', 'findlink', 'ips-agent',
'woriobot', 'mlbot', 'webbot', 'webcrawl',
'voilabot', 'rank/', 'ezooms', 'heritrix', 'indeedbot',