From 517bcc6197351c2fa731d511664c13050fc00a44 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 24 Oct 2021 19:05:04 +0100 Subject: [PATCH] Remove old user agent log entries --- daemon.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index d33e27113..fcfb9172a 100644 --- a/daemon.py +++ b/daemon.py @@ -408,7 +408,16 @@ class PubServer(BaseHTTPRequestHandler): "lastseen": currTime, "hits": 1 } - if currTime - self.server.lastKnownCrawler >= 10: + + if currTime - self.server.lastKnownCrawler >= 30: + # remove any old observations + removeCrawlers = [] + for ua, item in self.server.knownCrawlers.items(): + if currTime - item['lastseen'] >= 60 * 60 * 24 * 30: + removeCrawlers.append(ua) + for ua in removeCrawlers: + del self.server.knownCrawlers[ua] + # save the list of crawlers saveJson(self.server.knownCrawlers, self.server.baseDir + '/accounts/knownCrawlers.json') self.server.lastKnownCrawler = currTime