From 9391fa57c924ba0339f9d528567563c8e1059d2b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 19 Oct 2020 17:33:58 +0100 Subject: [PATCH] Option for maximum mirrored articles --- daemon.py | 6 +++++- epicyon.py | 19 ++++++++++++++----- newsdaemon.py | 22 ++++++++++++++++++++-- tests.py | 6 +++--- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/daemon.py b/daemon.py index f29556e5a..dbb8062cb 100644 --- a/daemon.py +++ b/daemon.py @@ -11981,7 +11981,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(maxNewswireFeedSizeKb: int, +def runDaemon(maxMirroredArticles: int, + maxNewswireFeedSizeKb: int, maxNewswirePostsPerSource: int, showPublishedDateOnly: bool, votingTimeMins: int, @@ -12116,6 +12117,9 @@ def runDaemon(maxNewswireFeedSizeKb: int, # Show only the date at the bottom of posts, and not the time httpd.showPublishedDateOnly = showPublishedDateOnly + # maximum number of news articles to mirror + httpd.maxMirroredArticles = maxMirroredArticles + if registration == 'open': httpd.registration = True else: diff --git a/epicyon.py b/epicyon.py index 5b4787869..427a53032 100644 --- a/epicyon.py +++ b/epicyon.py @@ -120,6 +120,11 @@ parser.add_argument('--maxFeedSize', dest='maxNewswireFeedSizeKb', type=int, default=2048, help='Maximum newswire rss/atom feed size in K') +parser.add_argument('--maxMirroredArticles', + dest='maxMirroredArticles', type=int, + default=100, + help='Maximum number of news articles to mirror.' + + ' Set to zero for indefinite mirroring.') parser.add_argument('--postcache', dest='maxRecentPosts', type=int, default=512, help='The maximum number of recent posts to store in RAM') @@ -1937,15 +1942,18 @@ if dateonly: maxNewswirePostsPerSource = \ getConfigParam(baseDir, 'maxNewswirePostsPerSource') if maxNewswirePostsPerSource: - if maxNewswirePostsPerSource.isdigit(): - args.maxNewswirePostsPerSource = maxNewswirePostsPerSource + args.maxNewswirePostsPerSource = int(maxNewswirePostsPerSource) # set the maximum size of a newswire rss/atom feed in Kilobytes maxNewswireFeedSizeKb = \ getConfigParam(baseDir, 'maxNewswireFeedSizeKb') if maxNewswireFeedSizeKb: - if maxNewswireFeedSizeKb.isdigit(): - args.maxNewswireFeedSizeKb = maxNewswireFeedSizeKb + args.maxNewswireFeedSizeKb = int(maxNewswireFeedSizeKb) + +maxMirroredArticles = \ + getConfigParam(baseDir, 'maxMirroredArticles') +if maxMirroredArticles is not None: + args.maxMirroredArticles = int(maxMirroredArticles) YTDomain = getConfigParam(baseDir, 'youtubedomain') if YTDomain: @@ -1960,7 +1968,8 @@ if setTheme(baseDir, themeName, domain): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.maxNewswireFeedSizeKb, + runDaemon(args.maxMirroredArticles, + args.maxNewswireFeedSizeKb, args.maxNewswirePostsPerSource, args.dateonly, args.votingtime, diff --git a/newsdaemon.py b/newsdaemon.py index 53a9686d0..7dd232f92 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -348,6 +348,17 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {}, return True +def createNewsMirror(baseDir: str, url: str, + maxMirroredArticles: int) -> bool: + """Creates a local mirror of a news article + """ + mirrorDir = baseDir + '/accounts/newsmirror' + if not os.path.isdir(mirrorDir): + os.mkdir(mirrorDir) + + return True + + def convertRSStoActivityPub(baseDir: str, httpPrefix: str, domain: str, port: int, newswire: {}, @@ -356,7 +367,8 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, session, cachedWebfingers: {}, personCache: {}, federationList: [], - sendThreads: [], postLog: []) -> None: + sendThreads: [], postLog: [], + maxMirroredArticles: int) -> None: """Converts rss items in a newswire into posts """ basePath = baseDir + '/accounts/news@' + domain + '/outbox' @@ -431,6 +443,11 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, if not blog: continue + mirrored = item[7] + if mirrored: + if not createNewsMirror(baseDir, url, maxMirroredArticles): + continue + idStr = \ httpPrefix + '://' + domain + '/users/news' + \ '/statuses/' + statusNumber + '/replies' @@ -571,7 +588,8 @@ def runNewswireDaemon(baseDir: str, httpd, httpd.personCache, httpd.federationList, httpd.sendThreads, - httpd.postLog) + httpd.postLog, + httpd.maxMirroredArticles) print('Newswire feed converted to ActivityPub') # wait a while before the next feeds update diff --git a/tests.py b/tests.py index 1f9d5b6ef..6e06fd9ab 100644 --- a/tests.py +++ b/tests.py @@ -290,7 +290,7 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(1024, 5, False, 0, False, 1, False, False, False, + runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -353,7 +353,7 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(1024, 5, False, 0, False, 1, False, False, False, + runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, @@ -390,7 +390,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(1024, 5, False, 0, False, 1, False, False, False, + runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port,