indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More LLM crawlers

main
bashrc 2026-08-30 14:54:25 +01:00
parent 6a0c77cbca
commit c23b110b54
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/crawlers.py
View File

@ -165,7 +165,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'azureai', 'kagi-fetcher', 'aranet-', 'apifybot', 'apifyweb',
'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae',
'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher',
'kimi-user', 'querit-searchbot', 'useai', 'cursor'
'kimi-user', 'querit-searchbot', 'useai', 'cursor',
'reflectionbot', 'exasearchbot', 'lightpanda'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: