More LLM user agents

main
Bob Mottram 2024-08-19 19:32:38 +01:00
parent caade2aa2b
commit 3852e8dbe7
1 changed files with 4 additions and 2 deletions

View File

@ -131,8 +131,10 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
# is this an LLM crawler?
llm_bot_strings = (
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
'anthropic', 'mlbot', 'claude-web', 'ccbot', 'facebookbot',
'google-extended', 'piplbot', 'oai-search'
'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',
'facebookbot', 'google-extended', 'piplbot', 'oai-search',
'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot',
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot'
)
for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: