indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Tidying

main
Bob Mottram 2025-11-15 13:57:25 +00:00
parent b02d542b62
commit cd0c3efada
1 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
crawlers.py
View File

@ -139,13 +139,12 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
llm_bot_strings = (
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
'anthropic', 'mlbot', 'claude-', 'claudebot', 'ccbot',
'piplbot', 'oai-search', 'meta-external',
'diffbot', 'perplexitybot', 'perplexity‑', 'novaact', 'operator',
'piplbot', 'oai-search', 'meta-external', 'diffbot',
'perplexitybot', 'perplexity‑', 'novaact', 'operator',
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler',
'icc-crawler', 'scrapy', 'timpibot',
'velenpublic', 'webzio-extended', 'cohere-ai',
'cohere-train', 'crawlspace',
'icc-crawler', 'scrapy', 'timpibot', 'velenpublic',
'webzio-extended', 'cohere-ai', 'cohere-train', 'crawlspace',
'img2dataset', 'imgproxy', 'isscyberriskcrawler', 'sidetrade',
'kangaroo.ai', 'kangaroo bot', 'iaskspider', 'duckassistbot',
'pangubot', 'semrush', 'poseidon research', 'awario',
@ -155,9 +154,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'bedrockbot', 'bigsur', 'bravebot', 'brightbot', 'buddybot',
'-autorag', 'cloudvertexbot', 'cotoyogi', 'deepseekbot',
'devin', 'echoboxbot', 'factset_spyderbot', 'firecrawlagent',
'iboubot', 'linerbot', 'linguee bot',
'meta-externalagent', 'meta-externalfetcher', 'meta-webindexer',
'mycentralaiscraperbot',
'iboubot', 'linerbot', 'linguee bot', 'meta-externalagent',
'meta-externalfetcher', 'meta-webindexer', 'mycentralaiscraperbot',
'openai', 'panscient', 'phindbot', 'qualifiedbot', 'quillbot',
'sbIntuitionsbot', 'semrushbot', 'shapbot', 'terracotta',
'velenpublicwebcrawler', 'wpbot', 'yak'