indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Simplify

main
Bob Mottram 2026-01-04 16:16:57 +00:00
parent 224dfc346f
commit 98c770347f
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawlers.py
View File

@ -148,7 +148,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'aibot', 'chatgpt',
'anthropic', 'mlbot', 'claude-', 'claudebot', 'ccbot',
'piplbot', 'oai-search', 'meta-external', 'diffbot',
'perplexitybot', 'perplexity‑', 'novaact', 'operator',
'perplexity', 'novaact', 'operator',
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler',
'icc-crawler', 'scrapy', 'timpibot', 'velenpublic',