indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More LLM crawlers

main
bashrc 2026-06-29 10:59:14 +01:00
parent 1bc4b1b697
commit 5f358a1005
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/crawlers.py
View File

@ -163,7 +163,9 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
'terra cotta', 'velenpublicwebcrawler', 'wpbot', 'yak', 'amzn-',
'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb',
'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae'
'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae',
'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher',
'kimi-user', 'querit-searchbot', 'useai', 'cursor'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: