mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add a flag to indicate that a newswire feed has its contents mirrored

main
Bob Mottram 2020-10-19 15:37:17 +01:00
parent 0f0a371521
commit 6411ffcd62
2 changed files with 31 additions and 25 deletions

4
daemon.py
View File

@ -11496,8 +11496,8 @@ class PubServer(BaseHTTPRequestHandler):
# removes a post # removes a post
if self.path.endswith('/rmpost'): if self.path.endswith('/rmpost'):
if '/users/' not in self.path: if '/users/' not in self.path:
print('ERROR: attempt to remove post was not authorized. ' + print('ERROR: attempt to remove post ' +
self.path) 'was not authorized. ' + self.path)
self._400() self._400()
self.server.POSTbusy = False self.server.POSTbusy = False
return return

52
newswire.py
View File

@ -81,7 +81,7 @@ def addNewswireDictEntry(baseDir: str, domain: str,
title: str, link: str, title: str, link: str,
votesStatus: str, postFilename: str, votesStatus: str, postFilename: str,
description: str, moderated: bool, description: str, moderated: bool,
tags=[]) -> None: mirrored: bool, tags=[]) -> None:
"""Update the newswire dictionary """Update the newswire dictionary
""" """
allText = title + ' ' + description allText = title + ' ' + description
@ -103,12 +103,13 @@ def addNewswireDictEntry(baseDir: str, domain: str,
postFilename, postFilename,
description, description,
moderated, moderated,
tags tags,
mirrored
] ]
def xml2StrToDict(baseDir: str, domain: str, def xml2StrToDict(baseDir: str, domain: str, xmlStr: str,
xmlStr: str, moderated: bool, moderated: bool, mirrored: bool,
maxPostsPerSource: int) -> {}: maxPostsPerSource: int) -> {}:
"""Converts an xml 2.0 string to a dictionary """Converts an xml 2.0 string to a dictionary
""" """
@ -157,7 +158,7 @@ def xml2StrToDict(baseDir: str, domain: str,
result, str(publishedDate), result, str(publishedDate),
title, link, title, link,
votesStatus, postFilename, votesStatus, postFilename,
description, moderated) description, moderated, mirrored)
postCtr += 1 postCtr += 1
if postCtr >= maxPostsPerSource: if postCtr >= maxPostsPerSource:
break break
@ -175,7 +176,7 @@ def xml2StrToDict(baseDir: str, domain: str,
str(publishedDate) + '+00:00', str(publishedDate) + '+00:00',
title, link, title, link,
votesStatus, postFilename, votesStatus, postFilename,
description, moderated) description, moderated, mirrored)
postCtr += 1 postCtr += 1
if postCtr >= maxPostsPerSource: if postCtr >= maxPostsPerSource:
break break
@ -186,8 +187,8 @@ def xml2StrToDict(baseDir: str, domain: str,
return result return result
def atomFeedToDict(baseDir: str, domain: str, def atomFeedToDict(baseDir: str, domain: str, xmlStr: str,
xmlStr: str, moderated: bool, moderated: bool, mirrored: bool,
maxPostsPerSource: int) -> {}: maxPostsPerSource: int) -> {}:
"""Converts an atom feed string to a dictionary """Converts an atom feed string to a dictionary
""" """
@ -236,7 +237,7 @@ def atomFeedToDict(baseDir: str, domain: str,
result, str(publishedDate), result, str(publishedDate),
title, link, title, link,
votesStatus, postFilename, votesStatus, postFilename,
description, moderated) description, moderated, mirrored)
postCtr += 1 postCtr += 1
if postCtr >= maxPostsPerSource: if postCtr >= maxPostsPerSource:
break break
@ -253,7 +254,7 @@ def atomFeedToDict(baseDir: str, domain: str,
str(publishedDate) + '+00:00', str(publishedDate) + '+00:00',
title, link, title, link,
votesStatus, postFilename, votesStatus, postFilename,
description, moderated) description, moderated, mirrored)
postCtr += 1 postCtr += 1
if postCtr >= maxPostsPerSource: if postCtr >= maxPostsPerSource:
break break
@ -264,24 +265,23 @@ def atomFeedToDict(baseDir: str, domain: str,
return result return result
def xmlStrToDict(baseDir: str, domain: str, def xmlStrToDict(baseDir: str, domain: str, xmlStr: str,
xmlStr: str, moderated: bool, moderated: bool, mirrored: bool,
maxPostsPerSource: int) -> {}: maxPostsPerSource: int) -> {}:
"""Converts an xml string to a dictionary """Converts an xml string to a dictionary
""" """
if 'rss version="2.0"' in xmlStr: if 'rss version="2.0"' in xmlStr:
return xml2StrToDict(baseDir, domain, return xml2StrToDict(baseDir, domain,
xmlStr, moderated, maxPostsPerSource) xmlStr, moderated, mirrored, maxPostsPerSource)
elif 'xmlns="http://www.w3.org/2005/Atom"' in xmlStr: elif 'xmlns="http://www.w3.org/2005/Atom"' in xmlStr:
return atomFeedToDict(baseDir, domain, return atomFeedToDict(baseDir, domain,
xmlStr, moderated, maxPostsPerSource) xmlStr, moderated, mirrored, maxPostsPerSource)
return {} return {}
def getRSS(baseDir: str, domain: str, def getRSS(baseDir: str, domain: str, session, url: str,
session, url: str, moderated: bool, moderated: bool, mirrored: bool,
maxPostsPerSource: int, maxPostsPerSource: int, maxFeedSizeKb: int) -> {}:
maxFeedSizeKb: int) -> {}:
"""Returns an RSS url as a dict """Returns an RSS url as a dict
""" """
if not isinstance(url, str): if not isinstance(url, str):
@ -307,8 +307,8 @@ def getRSS(baseDir: str, domain: str,
if result: if result:
if int(len(result.text) / 1024) < maxFeedSizeKb and \ if int(len(result.text) / 1024) < maxFeedSizeKb and \
not containsInvalidChars(result.text): not containsInvalidChars(result.text):
return xmlStrToDict(baseDir, domain, return xmlStrToDict(baseDir, domain, result.text,
result.text, moderated, moderated, mirrored,
maxPostsPerSource) maxPostsPerSource)
else: else:
print('WARN: feed is too large: ' + url) print('WARN: feed is too large: ' + url)
@ -463,7 +463,7 @@ def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
postJsonObject['object']['summary'], postJsonObject['object']['summary'],
postJsonObject['object']['url'], postJsonObject['object']['url'],
votes, fullPostFilename, votes, fullPostFilename,
description, moderated, description, moderated, False,
getHashtagsFromPost(postJsonObject)) getHashtagsFromPost(postJsonObject))
ctr += 1 ctr += 1
@ -549,8 +549,14 @@ def getDictFromNewswire(session, baseDir: str, domain: str,
moderated = True moderated = True
url = url.replace('*', '').strip() url = url.replace('*', '').strip()
itemsList = getRSS(baseDir, domain, # should this feed content be mirrored?
session, url, moderated, mirrored = False
if '!' in url:
mirrored = True
url = url.replace('!', '').strip()
itemsList = getRSS(baseDir, domain, session, url,
moderated, mirrored,
maxPostsPerSource, maxFeedSizeKb) maxPostsPerSource, maxFeedSizeKb)
for dateStr, item in itemsList.items(): for dateStr, item in itemsList.items():
result[dateStr] = item result[dateStr] = item