Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon

daemon.py

@@ -67,6 +67,7 @@ from person import removeAccount
 from person import canRemovePost
 from person import personSnooze
 from person import personUnsnooze
+from posts import removePostInteractions
 from posts import outboxMessageCreateWrap
 from posts import getPinnedPostAsJson
 from posts import pinPost
@@ -460,32 +461,6 @@ class PubServer(BaseHTTPRequestHandler):
         else:
             print('ERROR: unable to create vote')
 
-    def _removePostInteractions(self, postJsonObject: {}) -> None:
-        """Removes potentially sensitive interactions from a post
-        This is the type of thing which would be of interest to marketers
-        or of saleable value to them. eg. Knowing who likes who or what.
-        """
-        if postJsonObject.get('likes'):
-            postJsonObject['likes'] = {'items': []}
-
-        removeCollections = (
-            'shares', 'replies', 'bookmarks', 'ignores'
-        )
-        for removeName in removeCollections:
-            if postJsonObject.get(removeName):
-                postJsonObject[removeName] = {}
-
-        if not postJsonObject.get('object'):
-            return
-        if not isinstance(postJsonObject['object'], dict):
-            return
-        if postJsonObject['object'].get('likes'):
-            postJsonObject['object']['likes'] = {'items': []}
-
-        for removeName in removeCollections:
-            if postJsonObject['object'].get(removeName):
-                postJsonObject['object'][removeName] = {}
-
     def _requestHTTP(self) -> bool:
         """Should a http response be given?
         """
@@ -7676,7 +7651,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self._404()
                                     self.server.GETbusy = False
                                     return True
-                                self._removePostInteractions(pjo)
+                                removePostInteractions(pjo, True)
                             if self._requestHTTP():
                                 recentPostsCache = \
                                     self.server.recentPostsCache
@@ -7803,7 +7778,7 @@ class PubServer(BaseHTTPRequestHandler):
                         self._404()
                         self.server.GETbusy = False
                         return True
-                    self._removePostInteractions(pjo)
+                    removePostInteractions(pjo, True)
 
                 if self._requestHTTP():
                     recentPostsCache = \

posts.py

@@ -3036,6 +3036,92 @@ def _addPostToTimeline(filePath: str, boxname: str,
     return False
 
 
+def removePostInteractions(postJsonObject: {}, force: bool) -> bool:
+    """ Don't show likes, replies, bookmarks, DMs or shares (announces) to
+    unauthorized viewers. This makes the timeline less useful to
+    marketers and other surveillance-oriented organizations.
+    Returns False if this is a private post
+    """
+    hasObject = False
+    if postJsonObject.get('object'):
+        if isinstance(postJsonObject['object'], dict):
+            hasObject = True
+    if hasObject:
+        postObj = postJsonObject['object']
+        if not force:
+            # If not authorized and it's a private post
+            # then just don't show it within timelines
+            if not isPublicPost(postObj):
+                return False
+    else:
+        postObj = postJsonObject
+
+    # clear the likes
+    if postObj.get('likes'):
+        postObj['likes'] = {
+            'items': []
+        }
+    # remove other collections
+    removeCollections = (
+        'replies', 'shares', 'bookmarks', 'ignores'
+    )
+    for removeName in removeCollections:
+        if postObj.get(removeName):
+            postObj[removeName] = {}
+    return True
+
+
+def _passedNewswireVoting(newswireVotesThreshold: int,
+                          baseDir: str, domain: str,
+                          postFilename: str,
+                          positiveVoting: bool,
+                          votingTimeMins: int) -> bool:
+    """Returns true if the post has passed through newswire voting
+    """
+    # apply votes within this timeline
+    if newswireVotesThreshold <= 0:
+        return True
+    # note that the presence of an arrival file also indicates
+    # that this post is moderated
+    arrivalDate = \
+        locateNewsArrival(baseDir, domain, postFilename)
+    if not arrivalDate:
+        return True
+    # how long has elapsed since this post arrived?
+    currDate = datetime.datetime.utcnow()
+    timeDiffMins = \
+        int((currDate - arrivalDate).total_seconds() / 60)
+    # has the voting time elapsed?
+    if timeDiffMins < votingTimeMins:
+        # voting is still happening, so don't add this
+        # post to the timeline
+        return False
+    # if there a votes file for this post?
+    votesFilename = \
+        locateNewsVotes(baseDir, domain, postFilename)
+    if not votesFilename:
+        return True
+    # load the votes file and count the votes
+    votesJson = loadJson(votesFilename, 0, 2)
+    if not votesJson:
+        return True
+    if not positiveVoting:
+        if votesOnNewswireItem(votesJson) >= \
+           newswireVotesThreshold:
+            # Too many veto votes.
+            # Continue without incrementing
+            # the posts counter
+            return False
+    else:
+        if votesOnNewswireItem < \
+           newswireVotesThreshold:
+            # Not enough votes.
+            # Continue without incrementing
+            # the posts counter
+            return False
+    return True
+
+
 def _createBoxIndexed(recentPostsCache: {},
                       session, baseDir: str, boxname: str,
                       nickname: str, domain: str, port: int, httpPrefix: str,
@@ -3103,7 +3189,7 @@ def _createBoxIndexed(recentPostsCache: {},
     indexFilename = \
         baseDir + '/accounts/' + timelineNickname + '@' + domain + \
         '/' + indexBoxName + '.index'
-    postsCtr = 0
+    totalPostsCount = 0
     postsAddedToTimeline = 0
     if os.path.isfile(indexFilename):
         with open(indexFilename, 'r') as indexFile:
@@ -3114,47 +3200,17 @@ def _createBoxIndexed(recentPostsCache: {},
                 if not postFilename:
                     break
 
-                # apply votes within this timeline
+                # Has this post passed through the newswire voting stage?
-                if newswireVotesThreshold > 0:
+                if not _passedNewswireVoting(newswireVotesThreshold,
-                    # note that the presence of an arrival file also indicates
+                                             baseDir, domain,
-                    # that this post is moderated
+                                             postFilename,
-                    arrivalDate = \
+                                             positiveVoting,
-                        locateNewsArrival(baseDir, domain, postFilename)
+                                             votingTimeMins):
-                    if arrivalDate:
+                    continue
-                        # how long has elapsed since this post arrived?
-                        currDate = datetime.datetime.utcnow()
-                        timeDiffMins = \
-                            int((currDate - arrivalDate).total_seconds() / 60)
-                        # has the voting time elapsed?
-                        if timeDiffMins < votingTimeMins:
-                            # voting is still happening, so don't add this
-                            # post to the timeline
-                            continue
-                        # if there a votes file for this post?
-                        votesFilename = \
-                            locateNewsVotes(baseDir, domain, postFilename)
-                        if votesFilename:
-                            # load the votes file and count the votes
-                            votesJson = loadJson(votesFilename, 0, 2)
-                            if votesJson:
-                                if not positiveVoting:
-                                    if votesOnNewswireItem(votesJson) >= \
-                                       newswireVotesThreshold:
-                                        # Too many veto votes.
-                                        # Continue without incrementing
-                                        # the posts counter
-                                        continue
-                                else:
-                                    if votesOnNewswireItem < \
-                                       newswireVotesThreshold:
-                                        # Not enough votes.
-                                        # Continue without incrementing
-                                        # the posts counter
-                                        continue
 
                 # Skip through any posts previous to the current page
-                if postsCtr < int((pageNumber - 1) * itemsPerPage):
+                if totalPostsCount < int((pageNumber - 1) * itemsPerPage):
-                    postsCtr += 1
+                    totalPostsCount += 1
                     continue
 
                 # if this is a full path then remove the directories
@@ -3176,7 +3232,7 @@ def _createBoxIndexed(recentPostsCache: {},
                             if _addPostStringToTimeline(url,
                                                         boxname, postsInBox,
                                                         boxActor):
-                                postsCtr += 1
+                                totalPostsCount += 1
                                 postsAddedToTimeline += 1
                                 continue
 
@@ -3192,7 +3248,11 @@ def _createBoxIndexed(recentPostsCache: {},
                     if _addPostToTimeline(fullPostFilename, boxname,
                                           postsInBox, boxActor):
                         postsAddedToTimeline += 1
-                        postsCtr += 1
+                        totalPostsCount += 1
+                    else:
+                        print('WARN: Unable to add post ' + postUrl +
+                              ' nickname ' + nickname +
+                              ' timeline ' + boxname)
                 else:
                     if timelineNickname != nickname:
                         # if this is the features timeline
@@ -3203,7 +3263,11 @@ def _createBoxIndexed(recentPostsCache: {},
                             if _addPostToTimeline(fullPostFilename, boxname,
                                                   postsInBox, boxActor):
                                 postsAddedToTimeline += 1
-                                postsCtr += 1
+                                totalPostsCount += 1
+                            else:
+                                print('WARN: Unable to add features post ' +
+                                      postUrl + ' nickname ' + nickname +
+                                      ' timeline ' + boxname)
                         else:
                             print('WARN: features timeline. ' +
                                   'Unable to locate post ' + postUrl)
@@ -3211,13 +3275,13 @@ def _createBoxIndexed(recentPostsCache: {},
                         print('WARN: Unable to locate post ' + postUrl +
                               ' nickname ' + nickname)
 
-    if postsCtr < 3:
+    if totalPostsCount < 3:
         print('Posts added to json timeline ' + boxname + ': ' +
               str(postsAddedToTimeline))
 
     # Generate first and last entries within header
-    if postsCtr > 0:
+    if totalPostsCount > 0:
-        lastPage = int(postsCtr / itemsPerPage)
+        lastPage = int(totalPostsCount / itemsPerPage)
         if lastPage < 1:
             lastPage = 1
         boxHeader['last'] = \
@@ -3258,21 +3322,9 @@ def _createBoxIndexed(recentPostsCache: {},
         # created by individualPostAsHtml
         p['hasReplies'] = hasReplies
 
-        # Don't show likes, replies, bookmarks, DMs or shares (announces) to
-        # unauthorized viewers
         if not authorized:
-            if p.get('object'):
+            if not removePostInteractions(p, False):
-                if isinstance(p['object'], dict):
+                continue
-                    if not isPublicPost(p):
-                        continue
-                    if p['object'].get('likes'):
-                        p['object']['likes'] = {'items': []}
-                    removeCollections = {
-                        'replies', 'shares', 'bookmarks', 'ignores'
-                    }
-                    for removeName in removeCollections:
-                        if p['object'].get(removeName):
-                            p['object'][removeName] = {}
 
         boxItems['orderedItems'].append(p)
 

tests.py

@@ -21,6 +21,7 @@ from cache import getPersonFromCache
 from threads import threadWithTrace
 from daemon import runDaemon
 from session import createSession
+from posts import removePostInteractions
 from posts import getMentionedPeople
 from posts import validContentWarning
 from posts import deleteAllPosts
@@ -3591,9 +3592,42 @@ def testUpdateActor():
         shutil.rmtree(baseDir + '/.tests')
 
 
+def testRemovePostInteractions() -> None:
+    print('testRemovePostInteractions')
+    postJsonObject = {
+        "type": "Create",
+        "object": {
+            "to": ["#Public"],
+            "likes": {
+                "items": ["a", "b", "c"]
+            },
+            "replies": {
+                "replyStuff": ["a", "b", "c"]
+            },
+            "shares": {
+                "sharesStuff": ["a", "b", "c"]
+            },
+            "bookmarks": {
+                "bookmarksStuff": ["a", "b", "c"]
+            },
+            "ignores": {
+                "ignoresStuff": ["a", "b", "c"]
+            }
+        }
+    }
+    removePostInteractions(postJsonObject, True)
+    assert postJsonObject['object']['likes']['items'] == []
+    assert postJsonObject['object']['replies'] == {}
+    assert postJsonObject['object']['shares'] == {}
+    assert postJsonObject['object']['bookmarks'] == {}
+    assert postJsonObject['object']['ignores'] == {}
+    assert not removePostInteractions(postJsonObject, False)
+
+
 def runAllTests():
     print('Running tests...')
     testFunctions()
+    testRemovePostInteractions()
     testExtractPGPPublicKey()
     testEmojiImages()
     testCamelCaseSplit()

webapp_timeline.py

@@ -749,7 +749,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str,
               str(itemCtr) + ' ' + str(timelineJson['orderedItems']))
 
     # page down arrow
-    if itemCtr > 2:
+    if itemCtr > 0:
         tlStr += textModeSeparator
         tlStr += \
             '      <center>\n' + \