indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Load hashtag swarm from cached version

main
Bob Mottram 2021-10-20 14:33:34 +01:00
parent 2873c4705b
commit b4e4159e8b
3 changed files with 95 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
inbox.py
View File

@ -15,6 +15,7 @@ import random
from linked_data_sig import verifyJsonSignature
from languages import understoodPostLanguage
from like import updateLikesCollection
from utils import fileLastModified
from utils import hasObjectString
from utils import hasObjectStringObject
from utils import getReplyIntervalHours
@ -105,6 +106,7 @@ from announce import createAnnounce
from notifyOnPost import notifyWhenPersonPosts
from conversation import updateConversation
from content import validHashTag
from webapp_hashtagswarm import htmlHashTagSwarm
def _storeLastPostId(baseDir: str, nickname: str, domain: str,
@ -138,7 +140,53 @@ def _storeLastPostId(baseDir: str, nickname: str, domain: str,
pass
def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
def _updateCachedHashtagSwarm(baseDir: str, nickname: str, domain: str,
httpPrefix: str, domainFull: str,
translate: {}) -> bool:
"""Updates the hashtag swarm stored as a file
"""
cachedHashtagSwarmFilename = \
acctDir(baseDir, nickname, domain) + '/.hashtagSwarm'
saveSwarm = True
if os.path.isfile(cachedHashtagSwarmFilename):
lastModified = fileLastModified(cachedHashtagSwarmFilename)
modifiedDate = None
try:
modifiedDate = \
datetime.strptime(lastModified, "%Y-%m-%dT%H:%M:%SZ")
except BaseException:
print('WARN: unable to parse last modified cache date ' +
str(lastModified))
pass
if modifiedDate:
currDate = datetime.datetime.utcnow()
timeDiff = currDate - modifiedDate
diffMins = divmod(timeDiff.total_seconds(), 60)
if diffMins < 10:
# was saved recently, so don't save again
# This avoids too much disk I/O
saveSwarm = False
else:
print('Updating cached hashtag swarm, last changed ' +
str(diffMins) + ' minutes ago')
if saveSwarm:
actor = httpPrefix + '://' + domainFull + '/users/' + nickname
newSwarmStr = htmlHashTagSwarm(baseDir, actor, translate)
if newSwarmStr:
try:
with open(cachedHashtagSwarmFilename, 'w+') as fp:
fp.write(newSwarmStr)
return True
except BaseException:
print('WARN: unable to write cached hashtag swarm ' +
cachedHashtagSwarmFilename)
pass
return False
def storeHashTags(baseDir: str, nickname: str, domain: str,
httpPrefix: str, domainFull: str,
postJsonObject: {}, translate: {}) -> None:
"""Extracts hashtags from an incoming post and updates the
relevant tags files.
"""
@ -161,6 +209,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
hashtagCategories = getHashtagCategories(baseDir)
hashtagsCtr = 0
for tag in postJsonObject['object']['tag']:
if not tag.get('type'):
continue
@ -179,6 +228,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
daysDiff = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
daysSinceEpoch = daysDiff.days
tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n'
hashtagsCtr += 1
if not os.path.isfile(tagsFilename):
with open(tagsFilename, 'w+') as tagsFile:
tagsFile.write(tagline)
@ -203,6 +253,12 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
if categoryStr:
setHashtagCategory(baseDir, tagName, categoryStr, False)
# if some hashtags were found then recalculate the swarm
# ready for later display
if hashtagsCtr > 0:
_updateCachedHashtagSwarm(baseDir, nickname, domain,
httpPrefix, domainFull, translate)
def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
translate: {},
@ -1594,7 +1650,9 @@ def _receiveAnnounce(recentPostsCache: {},
if debug:
print('DEBUG: Announce post downloaded for ' +
messageJson['actor'] + ' -> ' + messageJson['object'])
storeHashTags(baseDir, nickname, postJsonObject)
storeHashTags(baseDir, nickname, domain,
httpPrefix, domainFull,
postJsonObject, translate)
# Try to obtain the actor for this person
# so that their avatar can be shown
lookupActor = None
@ -2925,7 +2983,9 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
_inboxUpdateCalendar(baseDir, handle, postJsonObject)
storeHashTags(baseDir, handleName, postJsonObject)
storeHashTags(baseDir, handleName, domain,
httpPrefix, domainFull,
postJsonObject, translate)
# send the post out to group members
if isGroup:

23
newsdaemon.py
View File

@ -281,7 +281,8 @@ def hashtagRuleTree(operators: [],
def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
postJsonObject: {},
actionStr: str, hashtags: [], systemLanguage: str) -> None:
actionStr: str, hashtags: [], systemLanguage: str,
translate: {}) -> None:
"""Adds a hashtag via a hashtag rule
"""
addHashtag = actionStr.split('add ', 1)[1].strip()
@ -326,7 +327,12 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
else:
content += hashtagHtml
postJsonObject['object']['content'] = content
storeHashTags(baseDir, 'news', postJsonObject)
domain = domainFull
if ':' in domain:
domain = domain.split(':')[0]
storeHashTags(baseDir, 'news', domain,
httpPrefix, domainFull,
postJsonObject, translate)
def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
@ -369,7 +375,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
federationList: [],
sendThreads: [], postLog: [],
moderated: bool, url: str,
systemLanguage: str) -> bool:
systemLanguage: str,
translate: {}) -> bool:
"""Applies hashtag rules to a news post.
Returns true if the post should be saved to the news timeline
of this instance
@ -413,7 +420,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
if actionStr.startswith('add '):
# add a hashtag
_hashtagAdd(baseDir, httpPrefix, domainFull,
postJsonObject, actionStr, hashtags, systemLanguage)
postJsonObject, actionStr, hashtags, systemLanguage,
translate)
elif actionStr.startswith('remove '):
# remove a hashtag
_hashtagRemove(httpPrefix, domainFull, postJsonObject,
@ -659,7 +667,8 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
personCache, cachedWebfingers,
federationList,
sendThreads, postLog,
moderated, url, systemLanguage)
moderated, url, systemLanguage,
translate)
# save the post and update the index
if savePost:
@ -698,7 +707,9 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
if tag not in newswire[originalDateStr][6]:
newswire[originalDateStr][6].append(tag)
storeHashTags(baseDir, 'news', blog)
storeHashTags(baseDir, 'news', domain,
httpPrefix, domainFull,
blog, translate)
clearFromPostCaches(baseDir, recentPostsCache, postId)
if saveJson(blog, filename):

17
webapp_search.py
View File

@ -419,8 +419,21 @@ def htmlSearch(cssCache: {}, translate: {},
'name="submitSearch" accesskey="' + submitKey + '">' + \
translate['Submit'] + '</button>\n'
followStr += ' </form>\n'
followStr += ' <p class="hashtagswarm">' + \
htmlHashTagSwarm(baseDir, actor, translate) + '</p>\n'
cachedHashtagSwarmFilename = \
acctDir(baseDir, searchNickname, domain) + '/.hashtagSwarm'
swarmStr = ''
if os.path.isfile(cachedHashtagSwarmFilename):
try:
with open(cachedHashtagSwarmFilename, 'r') as fp:
swarmStr = fp.read()
except BaseException:
print('WARN: Unable to read cached hashtag swarm')
pass
if not swarmStr:
swarmStr = htmlHashTagSwarm(baseDir, actor, translate)
followStr += ' <p class="hashtagswarm">' + swarmStr + '</p>\n'
followStr += ' </center>\n'
followStr += ' </div>\n'
followStr += '</div>\n'