From ddadcf1f8baff8dc490ebcd63cda643ecceb045c Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@freedombone.net>
Date: Tue, 20 Oct 2020 10:27:58 +0100
Subject: [PATCH] Onion instances mirror via tor

---
 newsdaemon.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/newsdaemon.py b/newsdaemon.py
index d4f461fc..3b329759 100644
--- a/newsdaemon.py
+++ b/newsdaemon.py
@@ -350,7 +350,8 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
     return True
 
 
-def createNewsMirror(baseDir: str, postIdNumber: str, url: str,
+def createNewsMirror(baseDir: str, domain: str,
+                     postIdNumber: str, url: str,
                      maxMirroredArticles: int) -> bool:
     """Creates a local mirror of a news article
     """
@@ -408,14 +409,20 @@ def createNewsMirror(baseDir: str, postIdNumber: str, url: str,
         # already mirrored
         return True
 
+    # for onion instances mirror via tor
+    prefixStr = ''
+    if domain.endswith('.onion'):
+        prefixStr = '/usr/bin/torsocks '
+
     # download the files
     commandStr = \
-        '/usr/bin/wget -mkEpnp -e robots=off ' + url + \
+        prefixStr + '/usr/bin/wget -mkEpnp -e robots=off ' + url + \
         ' -P ' + mirrorArticleDir
     p = Popen(commandStr, shell=True)
     os.waitpid(p.pid, 0)
 
     if not os.path.isdir(mirrorArticleDir):
+        print('WARN: failed to mirror ' + url)
         return True
 
     # append the post Id number to the index file
@@ -528,7 +535,7 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
             continue
 
         if mirrored:
-            if not createNewsMirror(baseDir, statusNumber,
+            if not createNewsMirror(baseDir, domain, statusNumber,
                                     url, maxMirroredArticles):
                 continue