diff --git a/crawlers.py b/crawlers.py index 0d7e16997..2243d037b 100644 --- a/crawlers.py +++ b/crawlers.py @@ -128,7 +128,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str, # unless this is a news instance or if it is in the allowed list bot_strings = ('bot/', 'bot-', '/bot', '/robot', 'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'spider/', - 'externalhit/', 'chatgpt') + 'externalhit/', 'chatgpt', 'google', + 'anthropic') contains_bot_string = False for bot_str in bot_strings: if bot_str in agent_str_lower: