indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Comment

main
Bob Mottram 2024-09-03 13:52:04 +01:00
parent 01254fd764
commit 550e81d32d
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
crawlers.py
View File

@ -129,6 +129,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
llm = False
# is this an LLM crawler?
# https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt
llm_bot_strings = (
'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',