From 6208a3f00f4eecda5fb473064b9b634b1f8959b6 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 7 Oct 2020 14:51:29 +0100 Subject: [PATCH] Convert rss feed items to activitypub posts --- daemon.py | 4 ++- newsdaemon.py | 73 +++++++++++++++++++++++++++++++++++++++++++- newswire.py | 1 - posts.py | 3 +- translations/ar.json | 3 +- translations/ca.json | 3 +- translations/cy.json | 3 +- translations/de.json | 3 +- translations/en.json | 3 +- translations/es.json | 3 +- translations/fr.json | 3 +- translations/ga.json | 3 +- translations/hi.json | 3 +- translations/it.json | 3 +- translations/ja.json | 3 +- translations/oc.json | 3 +- translations/pt.json | 3 +- translations/ru.json | 3 +- translations/zh.json | 3 +- 19 files changed, 107 insertions(+), 19 deletions(-) diff --git a/daemon.py b/daemon.py index a36513705..0c23ee96f 100644 --- a/daemon.py +++ b/daemon.py @@ -11534,7 +11534,9 @@ def runDaemon(newsInstance: bool, print('Creating newswire thread') httpd.thrNewswireDaemon = \ threadWithTrace(target=runNewswireDaemon, - args=(baseDir, httpd, 'newswire'), daemon=True) + args=(baseDir, httpd, + httpPrefix, domain, port, + httpd.translate), daemon=True) # flags used when restarting the inbox queue httpd.restartInboxQueueInProgress = False diff --git a/newsdaemon.py b/newsdaemon.py index 708a7dacb..6df1600fa 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -6,11 +6,76 @@ __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" +import os import time from newswire import getDictFromNewswire +from posts import createNewsPost +from utils import saveJson -def runNewswireDaemon(baseDir: str, httpd, unused: str): +def updateFeedsIndex(baseDir: str, filename: str) -> None: + """Updates the index used for imported RSS feeds + """ + indexFilename = baseDir + '/accounts/feeds.index' + + if os.path.isfile(indexFilename): + if filename not in open(indexFilename).read(): + try: + with open(indexFilename, 'r+') as feedsFile: + content = feedsFile.read() + feedsFile.seek(0, 0) + feedsFile.write(filename + '\n' + content) + print('DEBUG: feeds post added to index') + except Exception as e: + print('WARN: Failed to write entry to feeds posts index ' + + indexFilename + ' ' + str(e)) + else: + feedsFile = open(indexFilename, 'w+') + if feedsFile: + feedsFile.write(filename + '\n') + feedsFile.close() + + +def convertRSStoActivityPub(baseDir: str, httpPrefix: str, + domain: str, port: int, + newswire: {}, + translate: {}) -> None: + """Converts rss items in a newswire into posts + """ + basePath = baseDir + '/accounts/feeds' + if not os.path.isdir(basePath): + os.mkdir(basePath) + + nickname = 'feeds' + + for dateStr, item in newswire.items(): + dateStr = dateStr.replace(' ', 'T') + dateStr = dateStr.replace('+00:00', 'Z') + + filename = basePath + '/' + dateStr + '.json' + if os.path.isfile(filename): + continue + + rssTitle = item[0] + url = item[1] + rssDescription = item[4] + if rssDescription: + rssDescription += \ + '\n\n' + translate['Read more...'] + '\n' + url + else: + rssDescription = url + blog = createNewsPost(baseDir, + nickname, domain, port, + httpPrefix, dateStr, + rssTitle, rssDescription, + None, None, None, False) + if saveJson(blog, filename): + updateFeedsIndex(baseDir, filename) + + +def runNewswireDaemon(baseDir: str, httpd, + httpPrefix: str, domain: str, port: int, + translate: {}) -> None: """Periodically updates RSS feeds """ # initial sleep to allow the system to start up @@ -33,6 +98,12 @@ def runNewswireDaemon(baseDir: str, httpd, unused: str): httpd.newswire = newNewswire print('Newswire updated') + + convertRSStoActivityPub(baseDir, + httpPrefix, domain, port, + newNewswire, translate) + print('Newswire feed converted to ActivityPub') + # wait a while before the next feeds update time.sleep(1200) diff --git a/newswire.py b/newswire.py index 1b86ef6ec..0d3ddceea 100644 --- a/newswire.py +++ b/newswire.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os -import time import requests from socket import error as SocketError import errno diff --git a/posts.py b/posts.py index 1e265ff6f..991fd3c84 100644 --- a/posts.py +++ b/posts.py @@ -1195,7 +1195,7 @@ def createBlogPost(baseDir: str, def createNewsPost(baseDir: str, nickname: str, domain: str, port: int, httpPrefix: str, - rssTitle: str, rssDescription: str, + published: str, rssTitle: str, rssDescription: str, attachImageFilename: str, mediaType: str, imageDescription: str, useBlurhash: bool) -> {}: """Converts title and description from an rss feed into a post @@ -1225,6 +1225,7 @@ def createNewsPost(baseDir: str, schedulePost, eventDate, eventTime, location) blog['object']['type'] = 'Article' + blog['object']['published'] = published return blog diff --git a/translations/ar.json b/translations/ar.json index f24ac3992..74ce376c6 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -304,5 +304,6 @@ "Vote": "تصويت", "Remove Vote": "إزالة التصويت", "This is a news instance": "هذا مثال أخبار", - "News": "أخبار" + "News": "أخبار", + "Read more...": "" } diff --git a/translations/ca.json b/translations/ca.json index d59073906..40f7b8154 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -304,5 +304,6 @@ "Vote": "Notícies", "Remove Vote": "Elimina el vot", "This is a news instance": "Aquesta és una instància de notícies", - "News": "Notícies" + "News": "Notícies", + "Read more...": "" } diff --git a/translations/cy.json b/translations/cy.json index 6129a816e..053d523be 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -304,5 +304,6 @@ "Vote": "Newyddion", "Remove Vote": "Tynnwch y Bleidlais", "This is a news instance": "Dyma enghraifft newyddion", - "News": "Newyddion" + "News": "Newyddion", + "Read more...": "" } diff --git a/translations/de.json b/translations/de.json index d0710d5a9..21c688451 100644 --- a/translations/de.json +++ b/translations/de.json @@ -304,5 +304,6 @@ "Vote": "Abstimmung", "Remove Vote": "Abstimmung entfernen", "This is a news instance": "Dies ist eine Nachrichteninstanz", - "News": "Nachrichten" + "News": "Nachrichten", + "Read more...": "" } diff --git a/translations/en.json b/translations/en.json index 420dfec52..36add8b2f 100644 --- a/translations/en.json +++ b/translations/en.json @@ -304,5 +304,6 @@ "Vote": "Vote", "Remove Vote": "Remove Vote", "This is a news instance": "This is a news instance", - "News": "News" + "News": "News", + "Read more...": "Read more..." } diff --git a/translations/es.json b/translations/es.json index a296bf69d..010138ebf 100644 --- a/translations/es.json +++ b/translations/es.json @@ -304,5 +304,6 @@ "Vote": "Votar", "Remove Vote": "Eliminar voto", "This is a news instance": "Esta es una instancia de noticias", - "News": "Noticias" + "News": "Noticias", + "Read more...": "" } diff --git a/translations/fr.json b/translations/fr.json index ec2955833..a9e3c739b 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -304,5 +304,6 @@ "Vote": "Voter", "Remove Vote": "Supprimer le vote", "This is a news instance": "Ceci est une instance d'actualité", - "News": "Nouvelles" + "News": "Nouvelles", + "Read more...": "" } diff --git a/translations/ga.json b/translations/ga.json index 49b67041d..429efb4d8 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -304,5 +304,6 @@ "Vote": "Vóta", "Remove Vote": "Bain Vóta", "This is a news instance": "Is sampla nuachta é seo", - "News": "Nuacht" + "News": "Nuacht", + "Read more...": "" } diff --git a/translations/hi.json b/translations/hi.json index b8e555726..428b8b1ec 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -304,5 +304,6 @@ "Vote": "वोट", "Remove Vote": "वोट हटा दें", "This is a news instance": "यह एक समाचार का उदाहरण है", - "News": "समाचार" + "News": "समाचार", + "Read more...": "" } diff --git a/translations/it.json b/translations/it.json index a8e22bd98..32788a6b8 100644 --- a/translations/it.json +++ b/translations/it.json @@ -304,5 +304,6 @@ "Vote": "Votazione", "Remove Vote": "Rimuovi voto", "This is a news instance": "Questa è un'istanza di notizie", - "News": "Notizia" + "News": "Notizia", + "Read more...": "" } diff --git a/translations/ja.json b/translations/ja.json index 299964631..b57fe63d9 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -304,5 +304,6 @@ "Vote": "投票", "Remove Vote": "投票を削除", "This is a news instance": "これはニュースインスタンスです", - "News": "ニュース" + "News": "ニュース", + "Read more...": "" } diff --git a/translations/oc.json b/translations/oc.json index 99eae9ad6..23780037b 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -300,5 +300,6 @@ "Vote": "Vote", "Remove Vote": "Remove Vote", "This is a news instance": "This is a news instance", - "News": "News" + "News": "News", + "Read more...": "Read more..." } diff --git a/translations/pt.json b/translations/pt.json index 015fa1859..67a076516 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -304,5 +304,6 @@ "Vote": "Voto", "Remove Vote": "Remover voto", "This is a news instance": "Esta é uma instância de notícias", - "News": "Notícia" + "News": "Notícia", + "Read more...": "" } diff --git a/translations/ru.json b/translations/ru.json index ca7623b34..726df9bea 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -304,5 +304,6 @@ "Vote": "Голос", "Remove Vote": "Удалить голос", "This is a news instance": "Это новостной экземпляр", - "News": "Новости" + "News": "Новости", + "Read more...": "" } diff --git a/translations/zh.json b/translations/zh.json index fb24bc136..dafc27164 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -304,5 +304,6 @@ "Vote": "投票", "Remove Vote": "删除投票", "This is a news instance": "这是一个新闻实例", - "News": "新闻" + "News": "新闻", + "Read more...": "" }