indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Ensure that crawler list is not saved too often

merge-requests/30/head
Bob Mottram 2021-10-24 12:20:31 +01:00
parent 0c4a3bf968
commit b54cc5b006
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
daemon.py
View File

@ -396,17 +396,19 @@ class PubServer(BaseHTTPRequestHandler):
"""Updates a dictionary of known crawlers accessing nodeinfo """Updates a dictionary of known crawlers accessing nodeinfo
or the masto API or the masto API
""" """
currTime = int(time.time())
if self.server.knownCrawlers.get(uaStr): if self.server.knownCrawlers.get(uaStr):
self.server.knownCrawlers[uaStr]['hits'] += 1 self.server.knownCrawlers[uaStr]['hits'] += 1
self.server.knownCrawlers[uaStr]['lastseen'] = \ self.server.knownCrawlers[uaStr]['lastseen'] = currTime
int(time.time())
else: else:
self.server.knownCrawlers[uaStr] = { self.server.knownCrawlers[uaStr] = {
"lastseen": int(time.time()), "lastseen": currTime,
"hits": 1 "hits": 1
} }
saveJson(self.server.knownCrawlers, if currTime - self.server.lastKnownCrawler >= 10:
self.server.baseDir + '/accounts/knownCrawlers.json') saveJson(self.server.knownCrawlers,
self.server.baseDir + '/accounts/knownCrawlers.json')
self.server.lastKnownCrawler = currTime
def _getInstanceUrl(self, callingDomain: str) -> str: def _getInstanceUrl(self, callingDomain: str) -> str:
"""Returns the URL for this instance """Returns the URL for this instance
@ -17374,6 +17376,8 @@ def runDaemon(listsEnabled: str,
knownCrawlersFilename = baseDir + '/accounts/knownCrawlers.json' knownCrawlersFilename = baseDir + '/accounts/knownCrawlers.json'
if os.path.isfile(knownCrawlersFilename): if os.path.isfile(knownCrawlersFilename):
httpd.knownCrawlers = loadJson(knownCrawlersFilename) httpd.knownCrawlers = loadJson(knownCrawlersFilename)
# when was the last crawler seen?
httpd.lastKnownCrawler = 0
if listsEnabled: if listsEnabled:
httpd.listsEnabled = listsEnabled httpd.listsEnabled = listsEnabled