Option to archive news posts

daemon.py

@@ -11981,7 +11981,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None:
                 tokensLookup[token] = nickname
 
 
-def runDaemon(maxMirroredArticles: int,
+def runDaemon(maxNewsPosts: int,
+              maxMirroredArticles: int,
               maxNewswireFeedSizeKb: int,
               maxNewswirePostsPerSource: int,
               showPublishedDateOnly: bool,
@@ -12120,6 +12121,9 @@ def runDaemon(maxMirroredArticles: int,
     # maximum number of news articles to mirror
     httpd.maxMirroredArticles = maxMirroredArticles
 
+    # maximum number of posts in the news timeline/outbox
+    httpd.maxNewsPosts = maxNewsPosts
+
     if registration == 'open':
         httpd.registration = True
     else:

epicyon.py

@@ -125,6 +125,11 @@ parser.add_argument('--maxMirroredArticles',
                     default=100,
                     help='Maximum number of news articles to mirror.' +
                     ' Set to zero for indefinite mirroring.')
+parser.add_argument('--maxNewsPosts',
+                    dest='maxNewsPosts', type=int,
+                    default=0,
+                    help='Maximum number of news timeline posts to keep. ' +
+                    'Zero for no expiry.')
 parser.add_argument('--postcache', dest='maxRecentPosts', type=int,
                     default=512,
                     help='The maximum number of recent posts to store in RAM')
@@ -1968,7 +1973,8 @@ if setTheme(baseDir, themeName, domain):
     print('Theme set to ' + themeName)
 
 if __name__ == "__main__":
-    runDaemon(args.maxMirroredArticles,
+    runDaemon(args.maxNewsPosts,
+              args.maxMirroredArticles,
               args.maxNewswireFeedSizeKb,
               args.maxNewswirePostsPerSource,
               args.dateonly,

newsdaemon.py

@@ -22,6 +22,7 @@ from collections import OrderedDict
 from newswire import getDictFromNewswire
 # from posts import sendSignedJson
 from posts import createNewsPost
+from posts import archivePostsForPerson
 from content import removeHtmlTag
 from content import dangerousMarkup
 from content import validHashTag
@@ -714,6 +715,16 @@ def runNewswireDaemon(baseDir: str, httpd,
                                 httpd.maxMirroredArticles)
         print('Newswire feed converted to ActivityPub')
 
+        if httpd.maxNewsPosts > 0:
+            archiveDir = baseDir + '/archive'
+            archiveSubdir = \
+                archiveDir + '/accounts/news@' + domain + '/outbox'
+            archivePostsForPerson(httpPrefix, 'news',
+                                  domain, baseDir, 'outbox',
+                                  archiveSubdir,
+                                  httpd.recentPostsCache,
+                                  httpd.maxNewsPosts)
+
         # wait a while before the next feeds update
         time.sleep(1200)
 

posts.py

@@ -3217,11 +3217,21 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,
         if not os.path.isfile(filePath):
             continue
         if archiveDir:
-            repliesPath = filePath.replace('.json', '.replies')
             archivePath = os.path.join(archiveDir, postFilename)
             os.rename(filePath, archivePath)
-            if os.path.isfile(repliesPath):
+
-                os.rename(repliesPath, archivePath)
+            extensions = ('replies', 'votes', 'arrived', 'muted')
+            for ext in extensions:
+                extPath = filePath.replace('.json', '.' + ext)
+                if os.path.isfile(extPath):
+                    os.rename(extPath,
+                              archivePath.replace('.json', '.' + ext))
+                else:
+                    extPath = filePath.replace('.json',
+                                               '.json.' + ext)
+                    if os.path.isfile(extPath):
+                        os.rename(extPath,
+                                  archivePath.replace('.json', '.json.' + ext))
         else:
             deletePost(baseDir, httpPrefix, nickname, domain,
                        filePath, False, recentPostsCache)

tests.py

@@ -290,7 +290,7 @@ def createServerAlice(path: str, domain: str, port: int,
     onionDomain = None
     i2pDomain = None
     print('Server running: Alice')
-    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,
@@ -353,7 +353,7 @@ def createServerBob(path: str, domain: str, port: int,
     onionDomain = None
     i2pDomain = None
     print('Server running: Bob')
-    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,
@@ -390,7 +390,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [],
     onionDomain = None
     i2pDomain = None
     print('Server running: Eve')
-    runDaemon(100, 1024, 5, False, 0, False, 1, False, False, False,
+    runDaemon(0, 100, 1024, 5, False, 0, False, 1, False, False, False,
               5, True, True, 'en', __version__,
               "instanceId", False, path, domain,
               onionDomain, i2pDomain, None, port, port,
@@ -2215,8 +2215,7 @@ def testHashtagRuleTree():
     assert str(tree) == str(['and', ['#foo'], ['from', ['"testsite.com"']]])
     assert str(tagsInConditions) == str(['#foo'])
     hashtags = ['#foo']
-    assert hashtagRuleResolve(tree, hashtags, moderated, content,
+    assert hashtagRuleResolve(tree, hashtags, moderated, content, url)
-                              'testsite.com')
     assert not hashtagRuleResolve(tree, hashtags, moderated, content,
                                   'othersite.net')
 

utils.py

@@ -762,20 +762,11 @@ def deletePost(baseDir: str, httpPrefix: str,
         # remove any attachment
         removeAttachment(baseDir, httpPrefix, domain, postJsonObject)
 
-        # remove any mute file
+        extensions = ('votes', 'arrived', 'muted')
-        muteFilename = postFilename + '.muted'
+        for ext in extensions:
-        if os.path.isfile(muteFilename):
+            extFilename = postFilename + '.' + ext
-            os.remove(muteFilename)
+            if os.path.isfile(extFilename):
-
+                os.remove(extFilename)
-        # remove any votes file
-        votesFilename = postFilename + '.votes'
-        if os.path.isfile(votesFilename):
-            os.remove(votesFilename)
-
-        # remove any arrived file
-        arrivedFilename = postFilename + '.arrived'
-        if os.path.isfile(arrivedFilename):
-            os.remove(arrivedFilename)
 
         # remove cached html version of the post
         cachedPostFilename = \