mirror of https://gitlab.com/bashrc2/epicyon
More LLM user agents
parent
caade2aa2b
commit
3852e8dbe7
|
@ -131,8 +131,10 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
|
|||
# is this an LLM crawler?
|
||||
llm_bot_strings = (
|
||||
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
|
||||
'anthropic', 'mlbot', 'claude-web', 'ccbot', 'facebookbot',
|
||||
'google-extended', 'piplbot', 'oai-search'
|
||||
'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',
|
||||
'facebookbot', 'google-extended', 'piplbot', 'oai-search',
|
||||
'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot',
|
||||
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot'
|
||||
)
|
||||
for bot_str in llm_bot_strings:
|
||||
if bot_str in agent_str_lower:
|
||||
|
|
Loading…
Reference in New Issue