indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

More efficient reading of tag index

merge-requests/30/head
Bob Mottram 2020-11-21 13:45:37 +00:00
parent d4536a9dd8
commit a30c1c1044
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
webapp_hashtagswarm.py
View File

@ -76,6 +76,7 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str:
currTime = datetime.utcnow() currTime = datetime.utcnow()
daysSinceEpoch = (currTime - datetime(1970, 1, 1)).days daysSinceEpoch = (currTime - datetime(1970, 1, 1)).days
daysSinceEpochStr = str(daysSinceEpoch) + ' ' daysSinceEpochStr = str(daysSinceEpoch) + ' '
daysSinceEpochStr2 = str(daysSinceEpoch - 1) + ' '
recently = daysSinceEpoch - 1 recently = daysSinceEpoch - 1
tagSwarm = [] tagSwarm = []
domainHistogram = {} domainHistogram = {}
@ -99,8 +100,12 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str:
hashTagName = f.split('.')[0] hashTagName = f.split('.')[0]
if isBlockedHashtag(baseDir, hashTagName): if isBlockedHashtag(baseDir, hashTagName):
continue continue
if daysSinceEpochStr not in open(tagsFilename).read(): with open(tagsFilename, 'r') as fp:
continue # only read one line, which saves time and memory
lastTag = fp.readline()
if not lastTag.startswith(daysSinceEpochStr):
if not lastTag.startswith(daysSinceEpochStr2):
continue
with open(tagsFilename, 'r') as tagsFile: with open(tagsFilename, 'r') as tagsFile:
while True: while True:
line = tagsFile.readline() line = tagsFile.readline()