Load hashtag swarm from cached version

inbox.py

@@ -15,6 +15,7 @@ import random
 from linked_data_sig import verifyJsonSignature
 from languages import understoodPostLanguage
 from like import updateLikesCollection
+from utils import fileLastModified
 from utils import hasObjectString
 from utils import hasObjectStringObject
 from utils import getReplyIntervalHours
@@ -105,6 +106,7 @@ from announce import createAnnounce
 from notifyOnPost import notifyWhenPersonPosts
 from conversation import updateConversation
 from content import validHashTag
+from webapp_hashtagswarm import htmlHashTagSwarm
 
 
 def _storeLastPostId(baseDir: str, nickname: str, domain: str,
@@ -138,7 +140,53 @@ def _storeLastPostId(baseDir: str, nickname: str, domain: str,
         pass
 
 
-def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
+def _updateCachedHashtagSwarm(baseDir: str, nickname: str, domain: str,
+                              httpPrefix: str, domainFull: str,
+                              translate: {}) -> bool:
+    """Updates the hashtag swarm stored as a file
+    """
+    cachedHashtagSwarmFilename = \
+        acctDir(baseDir, nickname, domain) + '/.hashtagSwarm'
+    saveSwarm = True
+    if os.path.isfile(cachedHashtagSwarmFilename):
+        lastModified = fileLastModified(cachedHashtagSwarmFilename)
+        modifiedDate = None
+        try:
+            modifiedDate = \
+                datetime.strptime(lastModified, "%Y-%m-%dT%H:%M:%SZ")
+        except BaseException:
+            print('WARN: unable to parse last modified cache date ' +
+                  str(lastModified))
+            pass
+        if modifiedDate:
+            currDate = datetime.datetime.utcnow()
+            timeDiff = currDate - modifiedDate
+            diffMins = divmod(timeDiff.total_seconds(), 60)
+            if diffMins < 10:
+                # was saved recently, so don't save again
+                # This avoids too much disk I/O
+                saveSwarm = False
+            else:
+                print('Updating cached hashtag swarm, last changed ' +
+                      str(diffMins) + ' minutes ago')
+    if saveSwarm:
+        actor = httpPrefix + '://' + domainFull + '/users/' + nickname
+        newSwarmStr = htmlHashTagSwarm(baseDir, actor, translate)
+        if newSwarmStr:
+            try:
+                with open(cachedHashtagSwarmFilename, 'w+') as fp:
+                    fp.write(newSwarmStr)
+                    return True
+            except BaseException:
+                print('WARN: unable to write cached hashtag swarm ' +
+                      cachedHashtagSwarmFilename)
+                pass
+    return False
+
+
+def storeHashTags(baseDir: str, nickname: str, domain: str,
+                  httpPrefix: str, domainFull: str,
+                  postJsonObject: {}, translate: {}) -> None:
     """Extracts hashtags from an incoming post and updates the
     relevant tags files.
     """
@@ -161,6 +209,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
 
     hashtagCategories = getHashtagCategories(baseDir)
 
+    hashtagsCtr = 0
     for tag in postJsonObject['object']['tag']:
         if not tag.get('type'):
             continue
@@ -179,6 +228,7 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
         daysDiff = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
         daysSinceEpoch = daysDiff.days
         tagline = str(daysSinceEpoch) + '  ' + nickname + '  ' + postUrl + '\n'
+        hashtagsCtr += 1
         if not os.path.isfile(tagsFilename):
             with open(tagsFilename, 'w+') as tagsFile:
                 tagsFile.write(tagline)
@@ -203,6 +253,12 @@ def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
             if categoryStr:
                 setHashtagCategory(baseDir, tagName, categoryStr, False)
 
+    # if some hashtags were found then recalculate the swarm
+    # ready for later display
+    if hashtagsCtr > 0:
+        _updateCachedHashtagSwarm(baseDir, nickname, domain,
+                                  httpPrefix, domainFull, translate)
+
 
 def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
                                translate: {},
@@ -1594,7 +1650,9 @@ def _receiveAnnounce(recentPostsCache: {},
         if debug:
             print('DEBUG: Announce post downloaded for ' +
                   messageJson['actor'] + ' -> ' + messageJson['object'])
-        storeHashTags(baseDir, nickname, postJsonObject)
+        storeHashTags(baseDir, nickname, domain,
+                      httpPrefix, domainFull,
+                      postJsonObject, translate)
         # Try to obtain the actor for this person
         # so that their avatar can be shown
         lookupActor = None
@@ -2925,7 +2983,9 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
 
             _inboxUpdateCalendar(baseDir, handle, postJsonObject)
 
-            storeHashTags(baseDir, handleName, postJsonObject)
+            storeHashTags(baseDir, handleName, domain,
+                          httpPrefix, domainFull,
+                          postJsonObject, translate)
 
             # send the post out to group members
             if isGroup:

newsdaemon.py

@@ -281,7 +281,8 @@ def hashtagRuleTree(operators: [],
 
 def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
                 postJsonObject: {},
-                actionStr: str, hashtags: [], systemLanguage: str) -> None:
+                actionStr: str, hashtags: [], systemLanguage: str,
+                translate: {}) -> None:
     """Adds a hashtag via a hashtag rule
     """
     addHashtag = actionStr.split('add ', 1)[1].strip()
@@ -326,7 +327,12 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
     else:
         content += hashtagHtml
     postJsonObject['object']['content'] = content
-    storeHashTags(baseDir, 'news', postJsonObject)
+    domain = domainFull
+    if ':' in domain:
+        domain = domain.split(':')[0]
+    storeHashTags(baseDir, 'news', domain,
+                  httpPrefix, domainFull,
+                  postJsonObject, translate)
 
 
 def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
@@ -369,7 +375,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
                                federationList: [],
                                sendThreads: [], postLog: [],
                                moderated: bool, url: str,
-                               systemLanguage: str) -> bool:
+                               systemLanguage: str,
+                               translate: {}) -> bool:
     """Applies hashtag rules to a news post.
     Returns true if the post should be saved to the news timeline
     of this instance
@@ -413,7 +420,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
         if actionStr.startswith('add '):
             # add a hashtag
             _hashtagAdd(baseDir, httpPrefix, domainFull,
-                        postJsonObject, actionStr, hashtags, systemLanguage)
+                        postJsonObject, actionStr, hashtags, systemLanguage,
+                        translate)
         elif actionStr.startswith('remove '):
             # remove a hashtag
             _hashtagRemove(httpPrefix, domainFull, postJsonObject,
@@ -659,7 +667,8 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
                                               personCache, cachedWebfingers,
                                               federationList,
                                               sendThreads, postLog,
-                                              moderated, url, systemLanguage)
+                                              moderated, url, systemLanguage,
+                                              translate)
 
         # save the post and update the index
         if savePost:
@@ -698,7 +707,9 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
                 if tag not in newswire[originalDateStr][6]:
                     newswire[originalDateStr][6].append(tag)
 
-            storeHashTags(baseDir, 'news', blog)
+            storeHashTags(baseDir, 'news', domain,
+                          httpPrefix, domainFull,
+                          blog, translate)
 
             clearFromPostCaches(baseDir, recentPostsCache, postId)
             if saveJson(blog, filename):

webapp_search.py

@@ -419,8 +419,21 @@ def htmlSearch(cssCache: {}, translate: {},
         'name="submitSearch" accesskey="' + submitKey + '">' + \
         translate['Submit'] + '</button>\n'
     followStr += '  </form>\n'
-    followStr += '  <p class="hashtagswarm">' + \
+
-        htmlHashTagSwarm(baseDir, actor, translate) + '</p>\n'
+    cachedHashtagSwarmFilename = \
+        acctDir(baseDir, searchNickname, domain) + '/.hashtagSwarm'
+    swarmStr = ''
+    if os.path.isfile(cachedHashtagSwarmFilename):
+        try:
+            with open(cachedHashtagSwarmFilename, 'r') as fp:
+                swarmStr = fp.read()
+        except BaseException:
+            print('WARN: Unable to read cached hashtag swarm')
+            pass
+    if not swarmStr:
+        swarmStr = htmlHashTagSwarm(baseDir, actor, translate)
+
+    followStr += '  <p class="hashtagswarm">' + swarmStr + '</p>\n'
     followStr += '  </center>\n'
     followStr += '  </div>\n'
     followStr += '</div>\n'