indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Onion instances mirror via tor

main
Bob Mottram 2020-10-20 10:27:58 +01:00
parent 06ab99a96c
commit ddadcf1f8b
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
newsdaemon.py
View File

@ -350,7 +350,8 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
return True return True
def createNewsMirror(baseDir: str, postIdNumber: str, url: str, def createNewsMirror(baseDir: str, domain: str,
postIdNumber: str, url: str,
maxMirroredArticles: int) -> bool: maxMirroredArticles: int) -> bool:
"""Creates a local mirror of a news article """Creates a local mirror of a news article
""" """
@ -408,14 +409,20 @@ def createNewsMirror(baseDir: str, postIdNumber: str, url: str,
# already mirrored # already mirrored
return True return True
# for onion instances mirror via tor
prefixStr = ''
if domain.endswith('.onion'):
prefixStr = '/usr/bin/torsocks '
# download the files # download the files
commandStr = \ commandStr = \
'/usr/bin/wget -mkEpnp -e robots=off ' + url + \ prefixStr + '/usr/bin/wget -mkEpnp -e robots=off ' + url + \
' -P ' + mirrorArticleDir ' -P ' + mirrorArticleDir
p = Popen(commandStr, shell=True) p = Popen(commandStr, shell=True)
os.waitpid(p.pid, 0) os.waitpid(p.pid, 0)
if not os.path.isdir(mirrorArticleDir): if not os.path.isdir(mirrorArticleDir):
print('WARN: failed to mirror ' + url)
return True return True
# append the post Id number to the index file # append the post Id number to the index file
@ -528,7 +535,7 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
continue continue
if mirrored: if mirrored:
if not createNewsMirror(baseDir, statusNumber, if not createNewsMirror(baseDir, domain, statusNumber,
url, maxMirroredArticles): url, maxMirroredArticles):
continue continue