From 550e81d32d342b5f064921d528c3e04e172dcc90 Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@libreserver.org>
Date: Tue, 3 Sep 2024 13:52:04 +0100
Subject: [PATCH] Comment

---
 crawlers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/crawlers.py b/crawlers.py
index d32d068ad..9c5a5cd73 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -129,6 +129,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
         llm = False
 
         # is this an LLM crawler?
+        # https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt
         llm_bot_strings = (
             'gptbot', '-ai/', ' ai/', '-ai ', ' ai ', 'chatgpt',
             'anthropic', 'mlbot', 'claude-web', 'claudebot', 'ccbot',