indymedia/epicyon
7
4 Fork 1
mirror of https://gitlab.com/bashrc2/epicyon synced 2026-10-31 01:51:15 +00:00
Code Issues 14 Releases Wiki Activity

Extra llm crawlers

Browse Source
This commit is contained in:
bashrc committed 2026-10-09 18:19:58 +01:00
1 parent 9fa6571502
commit 2234167582
1 file changed
+5 -1
Download Patch File Download Diff File Expand all files Collapse all files
src/crawlers.py
+5 -1
View File
@@ -167,7 +167,11 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'exabot', 'nagetbot', 'newsai', 'henkbot', 'opencode', 'trae',
'agenttimes', 'shap-user', 'cragcrawler', 'geisthaus-pagefetcher',
'kimi-user', 'querit-searchbot', 'useai', 'cursor',
'reflectionbot', 'exasearchbot', 'lightpanda', 'ilands.ai'
'reflectionbot', 'exasearchbot', 'lightpanda', 'ilands.ai',
'keenablebot', 'agentdatabot', 'bixelbot',
'cloudflarebrowserrenderingcrawler', 'kimi-agent', 'qodercli',
'doubaobot', 'erniebot', 'qwenbot', 'kimi-searchbot',
'kimibot', 'mistralai-training', 'oai-adsbot', 'diffbot-user'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: