From 947a13fba235316c17b1dfd3514ee49c29789c64 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 9 Oct 2020 13:15:20 +0100 Subject: [PATCH] Voting time period --- daemon.py | 66 ++++++++++++++++++++++++++++-------------- epicyon.py | 6 +++- newsdaemon.py | 24 +++++++++++++-- person.py | 5 ++-- posts.py | 77 +++++++++++++++++++++++++++++-------------------- tests.py | 6 ++-- utils.py | 32 +++++++++++++++++++- webinterface.py | 2 +- 8 files changed, 155 insertions(+), 63 deletions(-) diff --git a/daemon.py b/daemon.py index df0cd6016..fec45f864 100644 --- a/daemon.py +++ b/daemon.py @@ -6004,7 +6004,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'inbox', authorized, 0, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) if inboxFeed: if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -6034,7 +6035,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'inbox', authorized, 0, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -6123,7 +6125,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'dm', authorized, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if inboxDMFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -6149,7 +6152,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'dm', authorized, 0, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxDMs(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6227,7 +6231,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlreplies', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxRepliesFeed: inboxRepliesFeed = [] if self._requestHTTP(): @@ -6253,7 +6258,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlreplies', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxReplies(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6331,7 +6337,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInMediaFeed, 'tlmedia', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxMediaFeed: inboxMediaFeed = [] if self._requestHTTP(): @@ -6357,7 +6364,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInMediaFeed, 'tlmedia', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxMedia(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6435,7 +6443,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInBlogsFeed, 'tlblogs', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxBlogsFeed: inboxBlogsFeed = [] if self._requestHTTP(): @@ -6461,7 +6470,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInBlogsFeed, 'tlblogs', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxBlogs(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6541,7 +6551,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInNewsFeed, 'tlnews', True, self.server.newswireVotesThreshold, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) if not inboxNewsFeed: inboxNewsFeed = [] if self._requestHTTP(): @@ -6568,7 +6579,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, 'tlnews', True, self.server.newswireVotesThreshold, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) currNickname = path.split('/users/')[1] if '/' in currNickname: currNickname = currNickname.split('/')[0] @@ -6709,7 +6721,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlbookmarks', authorized, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if bookmarksFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -6736,7 +6749,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'tlbookmarks', authorized, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlBookmarks(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6816,7 +6830,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlevents', authorized, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) print('eventsFeed: ' + str(eventsFeed)) if eventsFeed: if self._requestHTTP(): @@ -6843,7 +6858,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'tlevents', authorized, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlEvents(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6916,7 +6932,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'outbox', authorized, self.server.newswireVotesThreshold, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) if outboxFeed: if self._requestHTTP(): nickname = \ @@ -6942,7 +6959,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInFeed, 'outbox', authorized, self.server.newswireVotesThreshold, - self.server.positiveVoting) + self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlOutbox(self.server.defaultTimeline, self.server.recentPostsCache, @@ -7007,7 +7025,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'moderation', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) if moderationFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -7032,7 +7051,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'moderation', True, - 0, self.server.positiveVoting) + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlModeration(self.server.defaultTimeline, self.server.recentPostsCache, @@ -11387,7 +11407,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(positiveVoting: bool, +def runDaemon(votingTimeMins: int, + positiveVoting: bool, newswireVotesThreshold: int, newsInstance: bool, blogsInstance: bool, @@ -11496,6 +11517,9 @@ def runDaemon(positiveVoting: bool, print('ERROR: no translations loaded from ' + translationsFile) sys.exit() + # For moderated newswire feeds this is the amount of time allowed + # for voting after the post arrives + httpd.votingTimeMins = votingTimeMins # on the newswire, whether moderators vote positively for items # or against them (veto) httpd.positiveVoting = positiveVoting diff --git a/epicyon.py b/epicyon.py index 3ece2fe69..bdafac365 100644 --- a/epicyon.py +++ b/epicyon.py @@ -260,6 +260,9 @@ parser.add_argument('--minimumvotes', dest='minimumvotes', type=int, default=1, help='Minimum number of votes to remove or add' + ' a newswire item') +parser.add_argument('--votingtime', dest='votingtime', type=int, + default=1440, + help='Time to vote on newswire items in minutes') parser.add_argument('--message', dest='message', type=str, default=None, help='Message content') @@ -1919,7 +1922,8 @@ if setTheme(baseDir, themeName): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.positivevoting, + runDaemon(args.votingtime, + args.positivevoting, args.minimumvotes, args.newsinstance, args.blogsinstance, args.mediainstance, diff --git a/newsdaemon.py b/newsdaemon.py index 36602fa73..45053a9e2 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -41,6 +41,15 @@ def updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: feedsFile.close() +def saveArrivedTime(baseDir: str, postFilename: str, arrived: str) -> None: + """Saves the time when an rss post arrived to a file + """ + arrivedFile = open(postFilename + '.arrived', 'w+') + if arrivedFile: + arrivedFile.write(arrived) + arrivedFile.close() + + def convertRSStoActivityPub(baseDir: str, httpPrefix: str, domain: str, port: int, newswire: {}, @@ -87,8 +96,7 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, rssDescription = '' # get the rss description if it exists - if len(item) >= 5: - rssDescription = item[4] + rssDescription = item[4] # add the off-site link to the description if rssDescription: @@ -132,9 +140,21 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, postId = newPostId.replace('/', '#') + moderated = item[5] + # save the post and update the index if saveJson(blog, filename): updateFeedsOutboxIndex(baseDir, domain, postId + '.json') + + # Save a file containing the time when the post arrived + # this can then later be used to construct the news timeline + # excluding items during the voting period + if moderated: + saveArrivedTime(baseDir, filename, blog['object']['arrived']) + else: + if os.path.isfile(filename + '.arrived'): + os.remove(filename + '.arrived') + # set the url newswire[originalDateStr][1] = \ '/users/news/statuses/' + statusNumber diff --git a/person.py b/person.py index d58064a17..e0ee43e7a 100644 --- a/person.py +++ b/person.py @@ -596,7 +596,8 @@ def personBoxJson(recentPostsCache: {}, session, baseDir: str, domain: str, port: int, path: str, httpPrefix: str, noOfItems: int, boxname: str, authorized: bool, - newswireVotesThreshold: int, positiveVoting: bool) -> {}: + newswireVotesThreshold: int, positiveVoting: bool, + votingTimeMins: int) -> {}: """Obtain the inbox/outbox/moderation feed for the given person """ if boxname != 'inbox' and boxname != 'dm' and \ @@ -673,7 +674,7 @@ def personBoxJson(recentPostsCache: {}, return createNewsTimeline(session, baseDir, nickname, domain, port, httpPrefix, noOfItems, headerOnly, newswireVotesThreshold, positiveVoting, - pageNumber) + votingTimeMins, pageNumber) elif boxname == 'tlblogs': return createBlogsTimeline(session, baseDir, nickname, domain, port, httpPrefix, noOfItems, headerOnly, diff --git a/posts.py b/posts.py index 648fbde60..75282a232 100644 --- a/posts.py +++ b/posts.py @@ -47,6 +47,7 @@ from utils import loadJson from utils import saveJson from utils import getConfigParam from utils import locateNewsVotes +from utils import locateNewsArrival from utils import votesOnNewswireItem from media import attachMedia from media import replaceYouTube @@ -2476,7 +2477,7 @@ def createInbox(recentPostsCache: {}, session, baseDir, 'inbox', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - 0, False, pageNumber) + 0, False, 0, pageNumber) def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2485,7 +2486,7 @@ def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlbookmarks', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, - True, 0, False, pageNumber) + True, 0, False, 0, pageNumber) def createEventsTimeline(recentPostsCache: {}, @@ -2495,7 +2496,7 @@ def createEventsTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'tlevents', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, - True, 0, False, pageNumber) + True, 0, False, 0, pageNumber) def createDMTimeline(recentPostsCache: {}, @@ -2505,7 +2506,7 @@ def createDMTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'dm', nickname, domain, port, httpPrefix, itemsPerPage, - headerOnly, True, 0, False, pageNumber) + headerOnly, True, 0, False, 0, pageNumber) def createRepliesTimeline(recentPostsCache: {}, @@ -2515,7 +2516,7 @@ def createRepliesTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'tlreplies', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - 0, False, pageNumber) + 0, False, 0, pageNumber) def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2524,7 +2525,7 @@ def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlblogs', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - 0, False, pageNumber) + 0, False, 0, pageNumber) def createMediaTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2533,18 +2534,19 @@ def createMediaTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlmedia', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - 0, False, pageNumber) + 0, False, 0, pageNumber) def createNewsTimeline(session, baseDir: str, nickname: str, domain: str, port: int, httpPrefix: str, itemsPerPage: int, headerOnly: bool, newswireVotesThreshold: int, - positiveVoting: bool, pageNumber=None) -> {}: + positiveVoting: bool, votingTimeMins: int, + pageNumber=None) -> {}: return createBoxIndexed({}, session, baseDir, 'outbox', 'news', domain, port, httpPrefix, itemsPerPage, headerOnly, True, newswireVotesThreshold, positiveVoting, - pageNumber) + votingTimeMins, pageNumber) def createOutbox(session, baseDir: str, nickname: str, domain: str, @@ -2554,7 +2556,7 @@ def createOutbox(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'outbox', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, authorized, - 0, False, pageNumber) + 0, False, 0, pageNumber) def createModeration(baseDir: str, nickname: str, domain: str, port: int, @@ -2848,7 +2850,7 @@ def createBoxIndexed(recentPostsCache: {}, nickname: str, domain: str, port: int, httpPrefix: str, itemsPerPage: int, headerOnly: bool, authorized: bool, newswireVotesThreshold: int, positiveVoting: bool, - pageNumber=None) -> {}: + votingTimeMins: int, pageNumber=None) -> {}: """Constructs the box feed for a person with the given nickname """ if not authorized or not pageNumber: @@ -2919,27 +2921,38 @@ def createBoxIndexed(recentPostsCache: {}, # apply votes within this timeline if newswireVotesThreshold > 0: - # if there a votes file for this post? - votesFilename = \ - locateNewsVotes(baseDir, domain, postFilename) - if votesFilename: - # load the votes file and count the votes - votesJson = loadJson(votesFilename, 0, 2) - if votesJson: - if not positiveVoting: - if votesOnNewswireItem(votesJson) >= \ - newswireVotesThreshold: - # Too many veto votes. - # Continue without incrementing the - # posts counter - continue - else: - if votesOnNewswireItem < \ - newswireVotesThreshold: - # Not enough votes. - # Continue without incrementing the - # posts counter - continue + # note that the presence of an arrival file also indicates + # that this post is moderated + arrivalDate = \ + locateNewsArrival(baseDir, domain, postFilename) + if arrivalDate: + # how long has elapsed since this post arrived? + currDate = datetime.datetime.now() + timeDiffMins = \ + int((currDate - arrivalDate).total_seconds() / 60) + # has the voting time elapsed? + if timeDiffMins > votingTimeMins: + # if there a votes file for this post? + votesFilename = \ + locateNewsVotes(baseDir, domain, postFilename) + if votesFilename: + # load the votes file and count the votes + votesJson = loadJson(votesFilename, 0, 2) + if votesJson: + if not positiveVoting: + if votesOnNewswireItem(votesJson) >= \ + newswireVotesThreshold: + # Too many veto votes. + # Continue without incrementing + # the posts counter + continue + else: + if votesOnNewswireItem < \ + newswireVotesThreshold: + # Not enough votes. + # Continue without incrementing + # the posts counter + continue # Skip through any posts previous to the current page if postsCtr < int((pageNumber - 1) * itemsPerPage): diff --git a/tests.py b/tests.py index 0de1c0c6d..ea93839e1 100644 --- a/tests.py +++ b/tests.py @@ -287,7 +287,7 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(False, 1, False, False, False, + runDaemon(0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -350,7 +350,7 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(False, 1, False, False, False, + runDaemon(0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -387,7 +387,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(False, 1, False, False, False, + runDaemon(0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, diff --git a/utils.py b/utils.py index c83b22432..933c875e8 100644 --- a/utils.py +++ b/utils.py @@ -526,10 +526,40 @@ def locateNewsVotes(baseDir: str, domain: str, else: postUrl = postUrl + '.json.votes' - accountDir = baseDir + '/accounts/news' + '@' + domain + '/' + accountDir = baseDir + '/accounts/news@' + domain + '/' postFilename = accountDir + 'outbox/' + postUrl if os.path.isfile(postFilename): return postFilename + + return None + + +def locateNewsArrival(baseDir: str, domain: str, + postUrl: str) -> str: + """Returns the arrival time for a news post + within the news user account + """ + postUrl = \ + postUrl.strip().replace('\n', '').replace('\r', '') + + # if this post in the shared inbox? + postUrl = removeIdEnding(postUrl.strip()).replace('/', '#') + + if postUrl.endswith('.json'): + postUrl = postUrl + '.arrival' + else: + postUrl = postUrl + '.json.arrival' + + accountDir = baseDir + '/accounts/news@' + domain + '/' + postFilename = accountDir + 'outbox/' + postUrl + if os.path.isfile(postFilename): + with open(postFilename, 'r') as arrivalFile: + arrival = arrivalFile.read() + if arrival: + arrivalDate = \ + datetime.strptime(arrival, "%Y-%m-%dT%H:%M:%SZ") + return arrivalDate + return None diff --git a/webinterface.py b/webinterface.py index a3b43a31d..136722db4 100644 --- a/webinterface.py +++ b/webinterface.py @@ -2862,7 +2862,7 @@ def htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, str(currPage), httpPrefix, 10, 'outbox', - authorized, 0, False) + authorized, 0, False, 0) if not outboxFeed: break if len(outboxFeed['orderedItems']) == 0: