indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Extra AI crawler

main
Bob Mottram 2024-11-17 15:15:55 +00:00
parent 6c693fa39e
commit a4ddfa9b55
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawlers.py
View File

@ -140,7 +140,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'googleother', 'icc-crawler', 'scrapy', 'timpibot', 'googleother', 'icc-crawler', 'scrapy', 'timpibot',
'velenpublic', 'webzio-extended', 'cohere-ai', 'facebookexternal', 'velenpublic', 'webzio-extended', 'cohere-ai', 'facebookexternal',
'img2dataset', 'isscyberriskcrawler', 'sidetrade', 'kangaroo.ai', 'img2dataset', 'isscyberriskcrawler', 'sidetrade', 'kangaroo.ai',
'kangaroo bot', 'iaskspider' 'kangaroo bot', 'iaskspider', 'duckassistbot'
) )
for bot_str in llm_bot_strings: for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: if bot_str in agent_str_lower: