From 32758ffe62d308eb93cc58562e23fd99fa3b52a8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 21 Oct 2020 11:39:09 +0100 Subject: [PATCH] Option to archive news posts --- daemon.py | 6 +++++- epicyon.py | 8 +++++++- newsdaemon.py | 11 +++++++++++ posts.py | 16 +++++++++++++--- tests.py | 9 ++++----- utils.py | 19 +++++-------------- 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/daemon.py b/daemon.py index dbb8062c..2008e000 100644 --- a/daemon.py +++ b/daemon.py @@ -11981,7 +11981,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(maxMirroredArticles: int, +def runDaemon(maxNewsPosts: int, + maxMirroredArticles: int, maxNewswireFeedSizeKb: int, maxNewswirePostsPerSource: int, showPublishedDateOnly: bool, @@ -12120,6 +12121,9 @@ def runDaemon(maxMirroredArticles: int, # maximum number of news articles to mirror httpd.maxMirroredArticles = maxMirroredArticles + # maximum number of posts in the news timeline/outbox + httpd.maxNewsPosts = maxNewsPosts + if registration == 'open': httpd.registration = True else: diff --git a/epicyon.py b/epicyon.py index 427a5303..0b980f4c 100644 --- a/epicyon.py +++ b/epicyon.py @@ -125,6 +125,11 @@ parser.add_argument('--maxMirroredArticles', default=100, help='Maximum number of news articles to mirror.' + ' Set to zero for indefinite mirroring.') +parser.add_argument('--maxNewsPosts', + dest='maxNewsPosts', type=int, + default=0, + help='Maximum number of news timeline posts to keep. ' + + 'Zero for no expiry.') parser.add_argument('--postcache', dest='maxRecentPosts', type=int, default=512, help='The maximum number of recent posts to store in RAM') @@ -1968,7 +1973,8 @@ if setTheme(baseDir, themeName, domain): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.maxMirroredArticles, + runDaemon(args.maxNewsPosts, + args.maxMirroredArticles, args.maxNewswireFeedSizeKb, args.maxNewswirePostsPerSource, args.dateonly, diff --git a/newsdaemon.py b/newsdaemon.py index aed9ff20..3de4dce6 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -22,6 +22,7 @@ from collections import OrderedDict from newswire import getDictFromNewswire # from posts import sendSignedJson from posts import createNewsPost +from posts import archivePostsForPerson from content import removeHtmlTag from content import dangerousMarkup from content import validHashTag @@ -714,6 +715,16 @@ def runNewswireDaemon(baseDir: str, httpd, httpd.maxMirroredArticles) print('Newswire feed converted to ActivityPub') + if httpd.maxNewsPosts > 0: + archiveDir = baseDir + '/archive' + archiveSubdir = \ + archiveDir + '/accounts/news@' + domain + '/outbox' + archivePostsForPerson(httpPrefix, 'news', + domain, baseDir, 'outbox', + archiveSubdir, + httpd.recentPostsCache, + httpd.maxNewsPosts) + # wait a while before the next feeds update time.sleep(1200) diff --git a/posts.py b/posts.py index cd85388d..e033957b 100644 --- a/posts.py +++ b/posts.py @@ -3217,11 +3217,21 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str, if not os.path.isfile(filePath): continue if archiveDir: - repliesPath = filePath.replace('.json', '.replies') archivePath = os.path.join(archiveDir, postFilename) os.rename(filePath, archivePath) - if os.path.isfile(repliesPath): - os.rename(repliesPath, archivePath) + + extensions = ('replies', 'votes', 'arrived', 'muted') + for ext in extensions: + extPath = filePath.replace('.json', '.' + ext) + if os.path.isfile(extPath): + os.rename(extPath, + archivePath.replace('.json', '.' + ext)) + else: + extPath = filePath.replace('.json', + '.json.' + ext) + if os.path.isfile(extPath): + os.rename(extPath, + archivePath.replace('.json', '.json.' + ext)) else: deletePost(baseDir, httpPrefix, nickname, domain, filePath, False, recentPostsCache) diff --git a/tests.py b/tests.py index 2e309c84..ed308150 100644 --- a/tests.py +++ b/tests.py @@ -290,7 +290,7 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, + runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -353,7 +353,7 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, + runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -390,7 +390,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, + runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -2215,8 +2215,7 @@ def testHashtagRuleTree(): assert str(tree) == str(['and', ['#foo'], ['from', ['"testsite.com"']]]) assert str(tagsInConditions) == str(['#foo']) hashtags = ['#foo'] - assert hashtagRuleResolve(tree, hashtags, moderated, content, - 'testsite.com') + assert hashtagRuleResolve(tree, hashtags, moderated, content, url) assert not hashtagRuleResolve(tree, hashtags, moderated, content, 'othersite.net') diff --git a/utils.py b/utils.py index a8a60a54..77e3162b 100644 --- a/utils.py +++ b/utils.py @@ -762,20 +762,11 @@ def deletePost(baseDir: str, httpPrefix: str, # remove any attachment removeAttachment(baseDir, httpPrefix, domain, postJsonObject) - # remove any mute file - muteFilename = postFilename + '.muted' - if os.path.isfile(muteFilename): - os.remove(muteFilename) - - # remove any votes file - votesFilename = postFilename + '.votes' - if os.path.isfile(votesFilename): - os.remove(votesFilename) - - # remove any arrived file - arrivedFilename = postFilename + '.arrived' - if os.path.isfile(arrivedFilename): - os.remove(arrivedFilename) + extensions = ('votes', 'arrived', 'muted') + for ext in extensions: + extFilename = postFilename + '.' + ext + if os.path.isfile(extFilename): + os.remove(extFilename) # remove cached html version of the post cachedPostFilename = \