indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Extra crawlers

main
bashrc 2026-05-03 14:35:26 +01:00
parent eee7c0db62
commit 0020ae63c9
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers.py
View File

@ -162,7 +162,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot',
'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai',
'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot'
'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot',
'nagetbot', 'newsai'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: