From bdd428439e20348227198985c40fd9da704a3b9c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 29 Apr 2021 21:16:48 +0100 Subject: [PATCH 1/6] At least one post for page down --- webapp_timeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_timeline.py b/webapp_timeline.py index df42ebed5..9500ffb6e 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -749,7 +749,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, str(itemCtr) + ' ' + str(timelineJson['orderedItems'])) # page down arrow - if itemCtr > 2: + if itemCtr > 0: tlStr += textModeSeparator tlStr += \ '
\n' + \ From 4a2137654fbb57119712e15a13cbb81171ce868c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 30 Apr 2021 10:17:22 +0100 Subject: [PATCH 2/6] More descriptive variable name --- posts.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/posts.py b/posts.py index 12d82a505..ccdd3786c 100644 --- a/posts.py +++ b/posts.py @@ -3103,7 +3103,7 @@ def _createBoxIndexed(recentPostsCache: {}, indexFilename = \ baseDir + '/accounts/' + timelineNickname + '@' + domain + \ '/' + indexBoxName + '.index' - postsCtr = 0 + totalPostsCount = 0 postsAddedToTimeline = 0 if os.path.isfile(indexFilename): with open(indexFilename, 'r') as indexFile: @@ -3153,8 +3153,8 @@ def _createBoxIndexed(recentPostsCache: {}, continue # Skip through any posts previous to the current page - if postsCtr < int((pageNumber - 1) * itemsPerPage): - postsCtr += 1 + if totalPostsCount < int((pageNumber - 1) * itemsPerPage): + totalPostsCount += 1 continue # if this is a full path then remove the directories @@ -3176,7 +3176,7 @@ def _createBoxIndexed(recentPostsCache: {}, if _addPostStringToTimeline(url, boxname, postsInBox, boxActor): - postsCtr += 1 + totalPostsCount += 1 postsAddedToTimeline += 1 continue @@ -3192,7 +3192,7 @@ def _createBoxIndexed(recentPostsCache: {}, if _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor): postsAddedToTimeline += 1 - postsCtr += 1 + totalPostsCount += 1 else: if timelineNickname != nickname: # if this is the features timeline @@ -3203,7 +3203,7 @@ def _createBoxIndexed(recentPostsCache: {}, if _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor): postsAddedToTimeline += 1 - postsCtr += 1 + totalPostsCount += 1 else: print('WARN: features timeline. ' + 'Unable to locate post ' + postUrl) @@ -3211,13 +3211,13 @@ def _createBoxIndexed(recentPostsCache: {}, print('WARN: Unable to locate post ' + postUrl + ' nickname ' + nickname) - if postsCtr < 3: + if totalPostsCount < 3: print('Posts added to json timeline ' + boxname + ': ' + str(postsAddedToTimeline)) # Generate first and last entries within header - if postsCtr > 0: - lastPage = int(postsCtr / itemsPerPage) + if totalPostsCount > 0: + lastPage = int(totalPostsCount / itemsPerPage) if lastPage < 1: lastPage = 1 boxHeader['last'] = \ From 3af7fbeda5aa2784cbadd3992a08dd071e5dced4 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 30 Apr 2021 10:24:56 +0100 Subject: [PATCH 3/6] Debug for timeline --- posts.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/posts.py b/posts.py index ccdd3786c..719390761 100644 --- a/posts.py +++ b/posts.py @@ -3193,6 +3193,10 @@ def _createBoxIndexed(recentPostsCache: {}, postsInBox, boxActor): postsAddedToTimeline += 1 totalPostsCount += 1 + else: + print('WARN: Unable to add post ' + postUrl + + ' nickname ' + nickname + + ' timeline ' + boxname) else: if timelineNickname != nickname: # if this is the features timeline @@ -3204,6 +3208,10 @@ def _createBoxIndexed(recentPostsCache: {}, postsInBox, boxActor): postsAddedToTimeline += 1 totalPostsCount += 1 + else: + print('WARN: Unable to add features post ' + + postUrl + ' nickname ' + nickname + + ' timeline ' + boxname) else: print('WARN: features timeline. ' + 'Unable to locate post ' + postUrl) From 6bb57f11f557c74d15b87ee14f7ea6327bdf27e8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 30 Apr 2021 10:48:39 +0100 Subject: [PATCH 4/6] Tidying --- posts.py | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/posts.py b/posts.py index 719390761..38fc9c00c 100644 --- a/posts.py +++ b/posts.py @@ -3036,6 +3036,34 @@ def _addPostToTimeline(filePath: str, boxname: str, return False +def _removePostAttributes(postJsonObject: {}, authorized: bool) -> bool: + """ Don't show likes, replies, bookmarks, DMs or shares (announces) to + unauthorized viewers. This makes the timeline less useful to + marketers and other surveillance-oriented organizations. + Returns False if this is a private post + """ + if authorized: + return True + if not postJsonObject.get('object'): + return True + if not isinstance(postJsonObject['object'], dict): + return True + # If it's not a public post then just don't show it + if not isPublicPost(postJsonObject): + return False + # clear the likes + if postJsonObject['object'].get('likes'): + postJsonObject['object']['likes'] = {'items': []} + # remove other collections + removeCollections = ( + 'replies', 'shares', 'bookmarks', 'ignores' + ) + for removeName in removeCollections: + if postJsonObject['object'].get(removeName): + postJsonObject['object'][removeName] = {} + return True + + def _createBoxIndexed(recentPostsCache: {}, session, baseDir: str, boxname: str, nickname: str, domain: str, port: int, httpPrefix: str, @@ -3266,21 +3294,8 @@ def _createBoxIndexed(recentPostsCache: {}, # created by individualPostAsHtml p['hasReplies'] = hasReplies - # Don't show likes, replies, bookmarks, DMs or shares (announces) to - # unauthorized viewers - if not authorized: - if p.get('object'): - if isinstance(p['object'], dict): - if not isPublicPost(p): - continue - if p['object'].get('likes'): - p['object']['likes'] = {'items': []} - removeCollections = { - 'replies', 'shares', 'bookmarks', 'ignores' - } - for removeName in removeCollections: - if p['object'].get(removeName): - p['object'][removeName] = {} + if not _removePostAttributes(p, authorized): + continue boxItems['orderedItems'].append(p) From aed2713c5cf8ce8b12ec15859af4f1a5d4ec37ad Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 30 Apr 2021 12:45:46 +0100 Subject: [PATCH 5/6] Test for removing post interactions --- daemon.py | 31 +++---------------------------- posts.py | 40 ++++++++++++++++++++++++---------------- tests.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 44 deletions(-) diff --git a/daemon.py b/daemon.py index 668ad4236..ad0a7fee9 100644 --- a/daemon.py +++ b/daemon.py @@ -67,6 +67,7 @@ from person import removeAccount from person import canRemovePost from person import personSnooze from person import personUnsnooze +from posts import removePostInteractions from posts import outboxMessageCreateWrap from posts import getPinnedPostAsJson from posts import pinPost @@ -460,32 +461,6 @@ class PubServer(BaseHTTPRequestHandler): else: print('ERROR: unable to create vote') - def _removePostInteractions(self, postJsonObject: {}) -> None: - """Removes potentially sensitive interactions from a post - This is the type of thing which would be of interest to marketers - or of saleable value to them. eg. Knowing who likes who or what. - """ - if postJsonObject.get('likes'): - postJsonObject['likes'] = {'items': []} - - removeCollections = ( - 'shares', 'replies', 'bookmarks', 'ignores' - ) - for removeName in removeCollections: - if postJsonObject.get(removeName): - postJsonObject[removeName] = {} - - if not postJsonObject.get('object'): - return - if not isinstance(postJsonObject['object'], dict): - return - if postJsonObject['object'].get('likes'): - postJsonObject['object']['likes'] = {'items': []} - - for removeName in removeCollections: - if postJsonObject['object'].get(removeName): - postJsonObject['object'][removeName] = {} - def _requestHTTP(self) -> bool: """Should a http response be given? """ @@ -7676,7 +7651,7 @@ class PubServer(BaseHTTPRequestHandler): self._404() self.server.GETbusy = False return True - self._removePostInteractions(pjo) + removePostInteractions(pjo, True) if self._requestHTTP(): recentPostsCache = \ self.server.recentPostsCache @@ -7803,7 +7778,7 @@ class PubServer(BaseHTTPRequestHandler): self._404() self.server.GETbusy = False return True - self._removePostInteractions(pjo) + removePostInteractions(pjo, True) if self._requestHTTP(): recentPostsCache = \ diff --git a/posts.py b/posts.py index 38fc9c00c..2d85445d6 100644 --- a/posts.py +++ b/posts.py @@ -3036,31 +3036,38 @@ def _addPostToTimeline(filePath: str, boxname: str, return False -def _removePostAttributes(postJsonObject: {}, authorized: bool) -> bool: +def removePostInteractions(postJsonObject: {}, force: bool) -> bool: """ Don't show likes, replies, bookmarks, DMs or shares (announces) to unauthorized viewers. This makes the timeline less useful to marketers and other surveillance-oriented organizations. Returns False if this is a private post """ - if authorized: - return True - if not postJsonObject.get('object'): - return True - if not isinstance(postJsonObject['object'], dict): - return True - # If it's not a public post then just don't show it - if not isPublicPost(postJsonObject): - return False + hasObject = False + if postJsonObject.get('object'): + if isinstance(postJsonObject['object'], dict): + hasObject = True + if hasObject: + postObj = postJsonObject['object'] + if not force: + # If not authorized and it's a private post + # then just don't show it within timelines + if not isPublicPost(postObj): + return False + else: + postObj = postJsonObject + # clear the likes - if postJsonObject['object'].get('likes'): - postJsonObject['object']['likes'] = {'items': []} + if postObj.get('likes'): + postObj['likes'] = { + 'items': [] + } # remove other collections removeCollections = ( 'replies', 'shares', 'bookmarks', 'ignores' ) for removeName in removeCollections: - if postJsonObject['object'].get(removeName): - postJsonObject['object'][removeName] = {} + if postObj.get(removeName): + postObj[removeName] = {} return True @@ -3294,8 +3301,9 @@ def _createBoxIndexed(recentPostsCache: {}, # created by individualPostAsHtml p['hasReplies'] = hasReplies - if not _removePostAttributes(p, authorized): - continue + if not authorized: + if not removePostInteractions(p, False): + continue boxItems['orderedItems'].append(p) diff --git a/tests.py b/tests.py index ddbe4f247..9c9213ce4 100644 --- a/tests.py +++ b/tests.py @@ -21,6 +21,7 @@ from cache import getPersonFromCache from threads import threadWithTrace from daemon import runDaemon from session import createSession +from posts import removePostInteractions from posts import getMentionedPeople from posts import validContentWarning from posts import deleteAllPosts @@ -3591,9 +3592,42 @@ def testUpdateActor(): shutil.rmtree(baseDir + '/.tests') +def testRemovePostInteractions() -> None: + print('testRemovePostInteractions') + postJsonObject = { + "type": "Create", + "object": { + "to": ["#Public"], + "likes": { + "items": ["a", "b", "c"] + }, + "replies": { + "replyStuff": ["a", "b", "c"] + }, + "shares": { + "sharesStuff": ["a", "b", "c"] + }, + "bookmarks": { + "bookmarksStuff": ["a", "b", "c"] + }, + "ignores": { + "ignoresStuff": ["a", "b", "c"] + } + } + } + removePostInteractions(postJsonObject, True) + assert postJsonObject['object']['likes']['items'] == [] + assert postJsonObject['object']['replies'] == {} + assert postJsonObject['object']['shares'] == {} + assert postJsonObject['object']['bookmarks'] == {} + assert postJsonObject['object']['ignores'] == {} + assert not removePostInteractions(postJsonObject, False) + + def runAllTests(): print('Running tests...') testFunctions() + testRemovePostInteractions() testExtractPGPPublicKey() testEmojiImages() testCamelCaseSplit() From 4826f234cceb716f87e6306d2685e5d3d9dcbc51 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 30 Apr 2021 14:24:33 +0100 Subject: [PATCH 6/6] Tidying of newswire voting logic --- posts.py | 95 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/posts.py b/posts.py index 2d85445d6..e54802509 100644 --- a/posts.py +++ b/posts.py @@ -3071,6 +3071,57 @@ def removePostInteractions(postJsonObject: {}, force: bool) -> bool: return True +def _passedNewswireVoting(newswireVotesThreshold: int, + baseDir: str, domain: str, + postFilename: str, + positiveVoting: bool, + votingTimeMins: int) -> bool: + """Returns true if the post has passed through newswire voting + """ + # apply votes within this timeline + if newswireVotesThreshold <= 0: + return True + # note that the presence of an arrival file also indicates + # that this post is moderated + arrivalDate = \ + locateNewsArrival(baseDir, domain, postFilename) + if not arrivalDate: + return True + # how long has elapsed since this post arrived? + currDate = datetime.datetime.utcnow() + timeDiffMins = \ + int((currDate - arrivalDate).total_seconds() / 60) + # has the voting time elapsed? + if timeDiffMins < votingTimeMins: + # voting is still happening, so don't add this + # post to the timeline + return False + # if there a votes file for this post? + votesFilename = \ + locateNewsVotes(baseDir, domain, postFilename) + if not votesFilename: + return True + # load the votes file and count the votes + votesJson = loadJson(votesFilename, 0, 2) + if not votesJson: + return True + if not positiveVoting: + if votesOnNewswireItem(votesJson) >= \ + newswireVotesThreshold: + # Too many veto votes. + # Continue without incrementing + # the posts counter + return False + else: + if votesOnNewswireItem < \ + newswireVotesThreshold: + # Not enough votes. + # Continue without incrementing + # the posts counter + return False + return True + + def _createBoxIndexed(recentPostsCache: {}, session, baseDir: str, boxname: str, nickname: str, domain: str, port: int, httpPrefix: str, @@ -3149,43 +3200,13 @@ def _createBoxIndexed(recentPostsCache: {}, if not postFilename: break - # apply votes within this timeline - if newswireVotesThreshold > 0: - # note that the presence of an arrival file also indicates - # that this post is moderated - arrivalDate = \ - locateNewsArrival(baseDir, domain, postFilename) - if arrivalDate: - # how long has elapsed since this post arrived? - currDate = datetime.datetime.utcnow() - timeDiffMins = \ - int((currDate - arrivalDate).total_seconds() / 60) - # has the voting time elapsed? - if timeDiffMins < votingTimeMins: - # voting is still happening, so don't add this - # post to the timeline - continue - # if there a votes file for this post? - votesFilename = \ - locateNewsVotes(baseDir, domain, postFilename) - if votesFilename: - # load the votes file and count the votes - votesJson = loadJson(votesFilename, 0, 2) - if votesJson: - if not positiveVoting: - if votesOnNewswireItem(votesJson) >= \ - newswireVotesThreshold: - # Too many veto votes. - # Continue without incrementing - # the posts counter - continue - else: - if votesOnNewswireItem < \ - newswireVotesThreshold: - # Not enough votes. - # Continue without incrementing - # the posts counter - continue + # Has this post passed through the newswire voting stage? + if not _passedNewswireVoting(newswireVotesThreshold, + baseDir, domain, + postFilename, + positiveVoting, + votingTimeMins): + continue # Skip through any posts previous to the current page if totalPostsCount < int((pageNumber - 1) * itemsPerPage):