indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Another LLM crawler (sigh)

main
Bob Mottram 2024-08-21 13:33:06 +01:00
parent 283a0f39ea
commit 2f0b527881
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers.py
View File

@ -134,7 +134,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot', 'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',
'facebookbot', 'google-extended', 'piplbot', 'oai-search', 'facebookbot', 'google-extended', 'piplbot', 'oai-search',
'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot', 'applebot-extended', 'meta-external', 'diffbot', 'perplexitybot',
'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot' 'omgili', 'imagesiftbot', 'bytespider', 'amazonbot', 'youbot',
'petalbot'
) )
for bot_str in llm_bot_strings: for bot_str in llm_bot_strings:
if bot_str in agent_str_lower: if bot_str in agent_str_lower: