From 6a9aea9a3c7ba460e4b4028943794301db3b5a5c Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@libreserver.org>
Date: Mon, 6 May 2024 23:24:48 +0100
Subject: [PATCH] duplicate

---
 crawlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crawlers.py b/crawlers.py
index 06460b35d..54383bc0d 100644
--- a/crawlers.py
+++ b/crawlers.py
@@ -131,7 +131,7 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
                        ' ai ', 'spider/', 'spider.ht', '/spider.', '-spider',
                        'externalhit/', 'chatgpt', 'google', 'anthropic',
                        'facebook', 'slurp', 'crawler', 'crawling', ' crawl ',
-                       'gigablast', 'archive.org', 'httrack', 'spider/',
+                       'gigablast', 'archive.org', 'httrack',
                        'spider-', ' spider ', 'findlink', 'ips-agent',
                        'woriobot', 'mlbot', 'webbot', 'webcrawl',
                        'voilabot', 'rank/', 'ezooms', 'heritrix', 'indeedbot',