indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More LLM user agents

merge-requests/30/head
Bob Mottram 2024-08-19 19:23:54 +01:00
parent 343b177622
commit caade2aa2b
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crawlers.py
View File

@ -131,7 +131,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
# is this an LLM crawler? # is this an LLM crawler?
llm_bot_strings = ( llm_bot_strings = (
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt', 'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
'anthropic', 'mlbot' 'anthropic', 'mlbot', 'claude-web', 'ccbot', 'facebookbot',
'google-extended', 'piplbot', 'oai-search'
) )
for bot_str in llm_bot_strings: for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: if bot_str in agent_str_lower:
@ -155,7 +156,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'woriobot', 'webbot', 'webcrawl', 'woriobot', 'webbot', 'webcrawl',
'voilabot', 'rank/', 'ezooms', 'heritrix', 'indeedbot', 'voilabot', 'rank/', 'ezooms', 'heritrix', 'indeedbot',
'woobot', 'infobot', 'viewbot', 'swimgbot', 'eright', 'woobot', 'infobot', 'viewbot', 'swimgbot', 'eright',
'apercite', 'bot (', 'summify', 'ccbot', 'linkfind', 'apercite', 'bot (', 'summify', 'linkfind',
'linkanalyze', 'analyzer', 'wotbox', 'ichiro', 'linkanalyze', 'analyzer', 'wotbox', 'ichiro',
'drupact', 'searchengine', 'coccoc', 'drupact', 'searchengine', 'coccoc',
'explorer/', 'explorer;', 'crystalsemantics', 'explorer/', 'explorer;', 'crystalsemantics',