indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More LLM crawlers

main
Bob Mottram 2025-04-29 17:15:35 +01:00
parent 786e4f9ab7
commit 28784cd7a2
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
crawlers.py
View File

@ -141,14 +141,15 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot', 'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',
'facebookbot', 'google-extended', 'piplbot', 'oai-search', 'facebookbot', 'google-extended', 'piplbot', 'oai-search',
'applebot', 'meta-external', 'diffbot', 'perplexitybot', 'applebot', 'meta-external', 'diffbot', 'perplexitybot',
'perplexity‑', 'novaact', 'operator',
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot', 'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler', 'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler',
'googleother', 'icc-crawler', 'scrapy', 'timpibot', 'googleother', 'icc-crawler', 'scrapy', 'timpibot',
'velenpublic', 'webzio-extended', 'cohere-ai', 'velenpublic', 'webzio-extended', 'cohere-ai',
'cohere-train', 'crawlspace', 'facebookexternal', 'cohere-train', 'crawlspace', 'facebookexternal',
'img2dataset', 'isscyberriskcrawler', 'sidetrade', 'kangaroo.ai', 'img2dataset', 'imgproxy', 'isscyberriskcrawler', 'sidetrade',
'kangaroo bot', 'iaskspider', 'duckassistbot', 'pangubot', 'kangaroo.ai', 'kangaroo bot', 'iaskspider', 'duckassistbot',
'semrush' 'pangubot', 'semrush'
) )
for bot_str in llm_bot_strings: for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: if bot_str in agent_str_lower: