indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More llm bots

main
bashrc 2026-05-14 10:56:13 +01:00
parent 0bd8b629e8
commit b38ec2649f
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
crawlers.py
View File

@ -136,7 +136,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'chatglm', 'cloudflare-autorag', 'crawl4ai',
'facebookbot', 'facebookexternalhit',
'google-cloudvertexbot', 'google-extended', 'googleother',
'google-firebase', 'notebooklm', 'googleagent',
'google-firebase', 'google-gemini', 'notebooklm', 'googleagent',
'iaskbot', 'imagespider', 'kunato', 'laion', 'lcc',
'linkupbot', 'manus-', 'poggio', 'tavily', 'twinagent',
'wrtnbot', 'yandexadditional', 'zanista',
@ -161,9 +161,9 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'meta-externalfetcher', 'meta-webindexer', 'aiscraper',
'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot',
'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-', 'azureai',
'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb', 'exabot',
'nagetbot', 'newsai'
'terra cotta', 'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-',
'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb',
'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: