Option for maximum mirrored articles

daemon.py

@@ -11981,7 +11981,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None:
                 tokensLookup[token] = nickname
 
 
-def runDaemon(maxNewswireFeedSizeKb: int,
+def runDaemon(maxMirroredArticles: int,
+              maxNewswireFeedSizeKb: int,
               maxNewswirePostsPerSource: int,
               showPublishedDateOnly: bool,
               votingTimeMins: int,
@@ -12116,6 +12117,9 @@ def runDaemon(maxNewswireFeedSizeKb: int,
     # Show only the date at the bottom of posts, and not the time
     httpd.showPublishedDateOnly = showPublishedDateOnly
 
+    # maximum number of news articles to mirror
+    httpd.maxMirroredArticles = maxMirroredArticles
+
     if registration == 'open':
         httpd.registration = True
     else:

epicyon.py

@@ -120,6 +120,11 @@ parser.add_argument('--maxFeedSize',
                     dest='maxNewswireFeedSizeKb', type=int,
                     default=2048,
                     help='Maximum newswire rss/atom feed size in K')
+parser.add_argument('--maxMirroredArticles',
+                    dest='maxMirroredArticles', type=int,
+                    default=100,
+                    help='Maximum number of news articles to mirror.' +
+                    ' Set to zero for indefinite mirroring.')
 parser.add_argument('--postcache', dest='maxRecentPosts', type=int,
                     default=512,
                     help='The maximum number of recent posts to store in RAM')
@@ -1937,15 +1942,18 @@ if dateonly:
 maxNewswirePostsPerSource = \
     getConfigParam(baseDir, 'maxNewswirePostsPerSource')
 if maxNewswirePostsPerSource:
-    if maxNewswirePostsPerSource.isdigit():
-        args.maxNewswirePostsPerSource = maxNewswirePostsPerSource
+    args.maxNewswirePostsPerSource = int(maxNewswirePostsPerSource)
 
 # set the maximum size of a newswire rss/atom feed in Kilobytes
 maxNewswireFeedSizeKb = \
     getConfigParam(baseDir, 'maxNewswireFeedSizeKb')
 if maxNewswireFeedSizeKb:
-    if maxNewswireFeedSizeKb.isdigit():
-        args.maxNewswireFeedSizeKb = maxNewswireFeedSizeKb
+    args.maxNewswireFeedSizeKb = int(maxNewswireFeedSizeKb)
+
+maxMirroredArticles = \
+    getConfigParam(baseDir, 'maxMirroredArticles')
+if maxMirroredArticles is not None:
+    args.maxMirroredArticles = int(maxMirroredArticles)
 
 YTDomain = getConfigParam(baseDir, 'youtubedomain')
 if YTDomain:
@@ -1960,7 +1968,8 @@ if setTheme(baseDir, themeName, domain):
     print('Theme set to ' + themeName)
 
 if __name__ == "__main__":
-    runDaemon(args.maxNewswireFeedSizeKb,
+    runDaemon(args.maxMirroredArticles,
+              args.maxNewswireFeedSizeKb,
               args.maxNewswirePostsPerSource,
               args.dateonly,
               args.votingtime,

newsdaemon.py

@@ -348,6 +348,17 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
     return True
 
 
+def createNewsMirror(baseDir: str, url: str,
+                     maxMirroredArticles: int) -> bool:
+    """Creates a local mirror of a news article
+    """
+    mirrorDir = baseDir + '/accounts/newsmirror'
+    if not os.path.isdir(mirrorDir):
+        os.mkdir(mirrorDir)
+
+    return True
+
+
 def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
                             domain: str, port: int,
                             newswire: {},
@@ -356,7 +367,8 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
                             session, cachedWebfingers: {},
                             personCache: {},
                             federationList: [],
-                            sendThreads: [], postLog: []) -> None:
+                            sendThreads: [], postLog: [],
+                            maxMirroredArticles: int) -> None:
     """Converts rss items in a newswire into posts
     """
     basePath = baseDir + '/accounts/news@' + domain + '/outbox'
@@ -431,6 +443,11 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
         if not blog:
             continue
 
+        mirrored = item[7]
+        if mirrored:
+            if not createNewsMirror(baseDir, url, maxMirroredArticles):
+                continue
+
         idStr = \
             httpPrefix + '://' + domain + '/users/news' + \
             '/statuses/' + statusNumber + '/replies'
@@ -571,7 +588,8 @@ def runNewswireDaemon(baseDir: str, httpd,
                                 httpd.personCache,
                                 httpd.federationList,
                                 httpd.sendThreads,
-                                httpd.postLog)
+                                httpd.postLog,
+                                httpd.maxMirroredArticles)
         print('Newswire feed converted to ActivityPub')
 
         # wait a while before the next feeds update

tests.py

@@ -290,7 +290,7 @@ def createServerAlice(path: str, domain: str, port: int,
     onionDomain = None
     i2pDomain = None
     print('Server running: Alice')
-    runDaemon(1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,
@@ -353,7 +353,7 @@ def createServerBob(path: str, domain: str, port: int,
     onionDomain = None
     i2pDomain = None
     print('Server running: Bob')
-    runDaemon(1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,
@@ -390,7 +390,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [],
     onionDomain = None
     i2pDomain = None
     print('Server running: Eve')
-    runDaemon(1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,