indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Prepare for hashtag processing

merge-requests/30/head
Bob Mottram 2020-10-16 22:33:18 +01:00
parent 5ed417138a
commit 18034d20fe
1 changed files with 55 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
newsdaemon.py
View File

@ -11,6 +11,7 @@ import time
import datetime import datetime
from collections import OrderedDict from collections import OrderedDict
from newswire import getDictFromNewswire from newswire import getDictFromNewswire
# from posts import sendSignedJson
from posts import createNewsPost from posts import createNewsPost
from content import removeHtmlTag from content import removeHtmlTag
from content import dangerousMarkup from content import dangerousMarkup
@ -70,13 +71,38 @@ def removeControlCharacters(content: str) -> str:
return content return content
def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
hashtags: str, httpPrefix: str,
domain: str, port: int,
personCache: {},
cachedWebfingers: {},
federationList: [],
sendThreads: [], postLog: []) -> bool:
"""Applies hashtag rules to a news post.
Returns true if the post should be saved to the news timeline
of this instance
"""
# TODO
# If routing to another instance
# sendSignedJson(postJsonObject: {}, session, baseDir: str,
# nickname: str, domain: str, port: int,
# toNickname: str, toDomain: str, toPort: int, cc: str,
# httpPrefix: str, False, False,
# federationList: [],
# sendThreads: [], postLog: [], cachedWebfingers: {},
# personCache: {}, False, __version__) -> int:
return True
def convertRSStoActivityPub(baseDir: str, httpPrefix: str, def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
domain: str, port: int, domain: str, port: int,
newswire: {}, newswire: {},
translate: {}, translate: {},
recentPostsCache: {}, maxRecentPosts: int, recentPostsCache: {}, maxRecentPosts: int,
session, cachedWebfingers: {}, session, cachedWebfingers: {},
personCache: {}) -> None: personCache: {},
federationList: [],
sendThreads: [], postLog: []) -> None:
"""Converts rss items in a newswire into posts """Converts rss items in a newswire into posts
""" """
basePath = baseDir + '/accounts/news@' + domain + '/outbox' basePath = baseDir + '/accounts/news@' + domain + '/outbox'
@ -175,24 +201,33 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str,
moderated = item[5] moderated = item[5]
hashtags = item[6]
savePost = newswireHashtagProcessing(session, baseDir, blog, hashtags,
httpPrefix, domain, port,
personCache, cachedWebfingers,
federationList,
sendThreads, postLog)
# save the post and update the index # save the post and update the index
if saveJson(blog, filename): if savePost:
updateFeedsOutboxIndex(baseDir, domain, postId + '.json') if saveJson(blog, filename):
updateFeedsOutboxIndex(baseDir, domain, postId + '.json')
# Save a file containing the time when the post arrived # Save a file containing the time when the post arrived
# this can then later be used to construct the news timeline # this can then later be used to construct the news timeline
# excluding items during the voting period # excluding items during the voting period
if moderated: if moderated:
saveArrivedTime(baseDir, filename, blog['object']['arrived']) saveArrivedTime(baseDir, filename,
else: blog['object']['arrived'])
if os.path.isfile(filename + '.arrived'): else:
os.remove(filename + '.arrived') if os.path.isfile(filename + '.arrived'):
os.remove(filename + '.arrived')
# set the url # set the url
newswire[originalDateStr][1] = \ newswire[originalDateStr][1] = \
'/users/news/statuses/' + statusNumber '/users/news/statuses/' + statusNumber
# set the filename # set the filename
newswire[originalDateStr][3] = filename newswire[originalDateStr][3] = filename
def mergeWithPreviousNewswire(oldNewswire: {}, newNewswire: {}) -> None: def mergeWithPreviousNewswire(oldNewswire: {}, newNewswire: {}) -> None:
@ -251,7 +286,10 @@ def runNewswireDaemon(baseDir: str, httpd,
httpd.maxRecentPosts, httpd.maxRecentPosts,
httpd.session, httpd.session,
httpd.cachedWebfingers, httpd.cachedWebfingers,
httpd.personCache) httpd.personCache,
httpd.federationList,
httpd.sendThreads,
httpd.postLog)
print('Newswire feed converted to ActivityPub') print('Newswire feed converted to ActivityPub')
# wait a while before the next feeds update # wait a while before the next feeds update