diff --git a/crawlers.py b/crawlers.py index 47d216461..8b1174e5c 100644 --- a/crawlers.py +++ b/crawlers.py @@ -148,7 +148,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str, 'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'aibot', 'chatgpt', 'anthropic', 'mlbot', 'claude-', 'claudebot', 'ccbot', 'piplbot', 'oai-search', 'meta-external', 'diffbot', - 'perplexitybot', 'perplexity‑', 'novaact', 'operator', + 'perplexity', 'novaact', 'operator', 'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot', 'petalbot', 'ai2bot', 'allenai', 'firecrawl', 'friendlycrawler', 'icc-crawler', 'scrapy', 'timpibot', 'velenpublic',