indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Improve handling of contentMap within posts

main
Bob Mottram 2021-07-18 15:15:16 +01:00
parent e3e1716f1f
commit 50096bff11
19 changed files with 245 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
blog.py
View File

@ -16,6 +16,7 @@ from webapp_utils import htmlHeaderWithBlogMarkup
from webapp_utils import htmlFooter from webapp_utils import htmlFooter
from webapp_utils import getPostAttachmentsAsHtml from webapp_utils import getPostAttachmentsAsHtml
from webapp_media import addEmbeddedElements from webapp_media import addEmbeddedElements
from utils import getContentFromPost
from utils import isAccountDir from utils import isAccountDir
from utils import removeHtml from utils import removeHtml
from utils import getConfigParam from utils import getConfigParam
@ -164,7 +165,8 @@ def _htmlBlogPostContent(authorized: bool,
postJsonObject: {}, postJsonObject: {},
handle: str, restrictToDomain: bool, handle: str, restrictToDomain: bool,
peertubeInstances: [], peertubeInstances: [],
blogSeparator='<hr>') -> str: systemLanguage: str,
blogSeparator: str = '<hr>') -> str:
"""Returns the content for a single blog post """Returns the content for a single blog post
""" """
linkedAuthor = False linkedAuthor = False
@ -235,9 +237,9 @@ def _htmlBlogPostContent(authorized: bool,
if attachmentStr: if attachmentStr:
blogStr += '<br><center>' + attachmentStr + '</center>' blogStr += '<br><center>' + attachmentStr + '</center>'
if postJsonObject['object'].get('content'): jsonContent = getContentFromPost(postJsonObject, systemLanguage)
contentStr = addEmbeddedElements(translate, if jsonContent:
postJsonObject['object']['content'], contentStr = addEmbeddedElements(translate, jsonContent,
peertubeInstances) peertubeInstances)
if postJsonObject['object'].get('tag'): if postJsonObject['object'].get('tag'):
contentStr = replaceEmojiFromTags(contentStr, contentStr = replaceEmojiFromTags(contentStr,
@ -312,7 +314,8 @@ def _htmlBlogPostRSS2(authorized: bool,
baseDir: str, httpPrefix: str, translate: {}, baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, domainFull: str, nickname: str, domain: str, domainFull: str,
postJsonObject: {}, postJsonObject: {},
handle: str, restrictToDomain: bool) -> str: handle: str, restrictToDomain: bool,
systemLanguage: str) -> str:
"""Returns the RSS version 2 feed for a single blog post """Returns the RSS version 2 feed for a single blog post
""" """
rssStr = '' rssStr = ''
@ -327,7 +330,7 @@ def _htmlBlogPostRSS2(authorized: bool,
pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
titleStr = postJsonObject['object']['summary'] titleStr = postJsonObject['object']['summary']
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content) description = firstParagraphFromString(content)
rssStr = ' <item>' rssStr = ' <item>'
rssStr += ' <title>' + titleStr + '</title>' rssStr += ' <title>' + titleStr + '</title>'
@ -343,7 +346,8 @@ def _htmlBlogPostRSS3(authorized: bool,
baseDir: str, httpPrefix: str, translate: {}, baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, domainFull: str, nickname: str, domain: str, domainFull: str,
postJsonObject: {}, postJsonObject: {},
handle: str, restrictToDomain: bool) -> str: handle: str, restrictToDomain: bool,
systemLanguage: str) -> str:
"""Returns the RSS version 3 feed for a single blog post """Returns the RSS version 3 feed for a single blog post
""" """
rssStr = '' rssStr = ''
@ -358,7 +362,7 @@ def _htmlBlogPostRSS3(authorized: bool,
pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
titleStr = postJsonObject['object']['summary'] titleStr = postJsonObject['object']['summary']
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content) description = firstParagraphFromString(content)
rssStr = 'title: ' + titleStr + '\n' rssStr = 'title: ' + titleStr + '\n'
rssStr += 'link: ' + messageLink + '\n' rssStr += 'link: ' + messageLink + '\n'
@ -379,10 +383,10 @@ def _htmlBlogRemoveCwButton(blogStr: str, translate: {}) -> str:
return blogStr return blogStr
def _getSnippetFromBlogContent(postJsonObject: {}) -> str: def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str:
"""Returns a snippet of text from the blog post as a preview """Returns a snippet of text from the blog post as a preview
""" """
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
if '<p>' in content: if '<p>' in content:
content = content.split('<p>', 1)[1] content = content.split('<p>', 1)[1]
if '</p>' in content: if '</p>' in content:
@ -412,7 +416,7 @@ def htmlBlogPost(authorized: bool,
getConfigParam(baseDir, 'instanceTitle') getConfigParam(baseDir, 'instanceTitle')
published = postJsonObject['object']['published'] published = postJsonObject['object']['published']
title = postJsonObject['object']['summary'] title = postJsonObject['object']['summary']
snippet = _getSnippetFromBlogContent(postJsonObject) snippet = _getSnippetFromBlogContent(postJsonObject, systemLanguage)
blogStr = htmlHeaderWithBlogMarkup(cssFilename, instanceTitle, blogStr = htmlHeaderWithBlogMarkup(cssFilename, instanceTitle,
httpPrefix, domainFull, nickname, httpPrefix, domainFull, nickname,
systemLanguage, published, systemLanguage, published,
@ -424,7 +428,7 @@ def htmlBlogPost(authorized: bool,
nickname, domain, nickname, domain,
domainFull, postJsonObject, domainFull, postJsonObject,
None, False, None, False,
peertubeInstances) peertubeInstances, systemLanguage)
# show rss links # show rss links
blogStr += '<p class="rssfeed">' blogStr += '<p class="rssfeed">'
@ -452,7 +456,7 @@ def htmlBlogPage(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {}, baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int, nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int, noOfItems: int, pageNumber: int,
peertubeInstances: []) -> str: peertubeInstances: [], systemLanguage: str) -> str:
"""Returns a html blog page containing posts """Returns a html blog page containing posts
""" """
if ' ' in nickname or '@' in nickname or \ if ' ' in nickname or '@' in nickname or \
@ -514,7 +518,8 @@ def htmlBlogPage(authorized: bool, session,
nickname, domain, nickname, domain,
domainFull, item, domainFull, item,
None, True, None, True,
peertubeInstances) peertubeInstances,
systemLanguage)
if len(timelineJson['orderedItems']) >= noOfItems: if len(timelineJson['orderedItems']) >= noOfItems:
blogStr += navigateStr blogStr += navigateStr
@ -542,7 +547,7 @@ def htmlBlogPageRSS2(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {}, baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int, nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int, noOfItems: int, pageNumber: int,
includeHeader: bool) -> str: includeHeader: bool, systemLanguage: str) -> str:
"""Returns an RSS version 2 feed containing posts """Returns an RSS version 2 feed containing posts
""" """
if ' ' in nickname or '@' in nickname or \ if ' ' in nickname or '@' in nickname or \
@ -585,7 +590,7 @@ def htmlBlogPageRSS2(authorized: bool, session,
httpPrefix, translate, httpPrefix, translate,
nickname, domain, nickname, domain,
domainFull, item, domainFull, item,
None, True) None, True, systemLanguage)
if includeHeader: if includeHeader:
return blogRSS2 + rss2Footer() return blogRSS2 + rss2Footer()
@ -596,7 +601,8 @@ def htmlBlogPageRSS2(authorized: bool, session,
def htmlBlogPageRSS3(authorized: bool, session, def htmlBlogPageRSS3(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {}, baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int, nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int) -> str: noOfItems: int, pageNumber: int,
systemLanguage: str) -> str:
"""Returns an RSS version 3 feed containing posts """Returns an RSS version 3 feed containing posts
""" """
if ' ' in nickname or '@' in nickname or \ if ' ' in nickname or '@' in nickname or \
@ -630,7 +636,8 @@ def htmlBlogPageRSS3(authorized: bool, session,
httpPrefix, translate, httpPrefix, translate,
nickname, domain, nickname, domain,
domainFull, item, domainFull, item,
None, True) None, True,
systemLanguage)
return blogRSS3 return blogRSS3
@ -670,7 +677,7 @@ def htmlBlogView(authorized: bool,
session, baseDir: str, httpPrefix: str, session, baseDir: str, httpPrefix: str,
translate: {}, domain: str, port: int, translate: {}, domain: str, port: int,
noOfItems: int, noOfItems: int,
peertubeInstances: []) -> str: peertubeInstances: [], systemLanguage: str) -> str:
"""Show the blog main page """Show the blog main page
""" """
blogStr = '' blogStr = ''
@ -688,7 +695,8 @@ def htmlBlogView(authorized: bool,
return htmlBlogPage(authorized, session, return htmlBlogPage(authorized, session,
baseDir, httpPrefix, translate, baseDir, httpPrefix, translate,
nickname, domain, port, nickname, domain, port,
noOfItems, 1, peertubeInstances) noOfItems, 1, peertubeInstances,
systemLanguage)
domainFull = getFullDomain(domain, port) domainFull = getFullDomain(domain, port)
@ -714,7 +722,7 @@ def htmlEditBlog(mediaInstance: bool, translate: {},
path: str, path: str,
pageNumber: int, pageNumber: int,
nickname: str, domain: str, nickname: str, domain: str,
postUrl: str) -> str: postUrl: str, systemLanguage: str) -> str:
"""Edit a blog post after it was created """Edit a blog post after it was created
""" """
postFilename = locatePost(baseDir, nickname, domain, postUrl) postFilename = locatePost(baseDir, nickname, domain, postUrl)
@ -832,7 +840,7 @@ def htmlEditBlog(mediaInstance: bool, translate: {},
placeholderMessage + '</label>' placeholderMessage + '</label>'
messageBoxHeight = 800 messageBoxHeight = 800
contentStr = postJsonObject['object']['content'] contentStr = getContentFromPost(postJsonObject, systemLanguage)
contentStr = contentStr.replace('<p>', '').replace('</p>', '\n') contentStr = contentStr.replace('<p>', '').replace('</p>', '\n')
editBlogForm += \ editBlogForm += \

60
daemon.py
View File

@ -208,6 +208,7 @@ from shares import addShare
from shares import removeShare from shares import removeShare
from shares import expireShares from shares import expireShares
from categories import setHashtagCategory from categories import setHashtagCategory
from utils import getContentFromPost
from utils import acctDir from utils import acctDir
from utils import getImageExtensionFromMimeType from utils import getImageExtensionFromMimeType
from utils import getImageMimeType from utils import getImageMimeType
@ -1110,7 +1111,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain, self.server.YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.allowLocalNetworkAccess, self.server.allowLocalNetworkAccess,
city) city, self.server.systemLanguage)
def _postToOutboxThread(self, messageJson: {}) -> bool: def _postToOutboxThread(self, messageJson: {}) -> bool:
"""Creates a thread to send a post """Creates a thread to send a post
@ -1308,7 +1309,8 @@ class PubServer(BaseHTTPRequestHandler):
headersDict, headersDict,
self.path, self.path,
self.server.debug, self.server.debug,
self.server.blockedCache) self.server.blockedCache,
self.server.systemLanguage)
if queueFilename: if queueFilename:
# add json to the queue # add json to the queue
if queueFilename not in self.server.inboxQueue: if queueFilename not in self.server.inboxQueue:
@ -1696,7 +1698,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain, self.server.domain,
self.server.port, self.server.port,
searchHandle, searchHandle,
self.server.debug) self.server.debug,
self.server.systemLanguage)
else: else:
msg = \ msg = \
htmlModerationInfo(self.server.cssCache, htmlModerationInfo(self.server.cssCache,
@ -2344,7 +2347,8 @@ class PubServer(BaseHTTPRequestHandler):
domain, domain,
self.server.port, self.server.port,
optionsActor, optionsActor,
self.server.debug).encode('utf-8') self.server.debug,
self.server.systemLanguage).encode('utf-8')
msglen = len(msg) msglen = len(msg)
self._set_headers('text/html', msglen, self._set_headers('text/html', msglen,
cookie, callingDomain) cookie, callingDomain)
@ -3910,6 +3914,8 @@ class PubServer(BaseHTTPRequestHandler):
newsPostTitle newsPostTitle
postJsonObject['object']['content'] = \ postJsonObject['object']['content'] = \
newsPostContent newsPostContent
contentMap = postJsonObject['object']['contentMap']
contentMap[self.server.systemLanguage] = newsPostContent
# update newswire # update newswire
pubDate = postJsonObject['object']['published'] pubDate = postJsonObject['object']['published']
publishedDate = \ publishedDate = \
@ -5619,7 +5625,8 @@ class PubServer(BaseHTTPRequestHandler):
domain, domain,
port, port,
maxPostsInRSSFeed, 1, maxPostsInRSSFeed, 1,
True) True,
self.server.systemLanguage)
if msg is not None: if msg is not None:
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -5674,7 +5681,8 @@ class PubServer(BaseHTTPRequestHandler):
domain, domain,
port, port,
maxPostsInRSSFeed, 1, maxPostsInRSSFeed, 1,
False) False,
self.server.systemLanguage)
break break
if msg: if msg:
msg = rss2Header(httpPrefix, msg = rss2Header(httpPrefix,
@ -5776,7 +5784,7 @@ class PubServer(BaseHTTPRequestHandler):
baseDir: str, httpPrefix: str, baseDir: str, httpPrefix: str,
domain: str, port: int, proxyType: str, domain: str, port: int, proxyType: str,
GETstartTime, GETtimings: {}, GETstartTime, GETtimings: {},
debug: bool) -> None: debug: bool, systemLanguage: str) -> None:
"""Returns an RSS3 feed """Returns an RSS3 feed
""" """
nickname = path.split('/blog/')[1] nickname = path.split('/blog/')[1]
@ -5800,7 +5808,8 @@ class PubServer(BaseHTTPRequestHandler):
baseDir, httpPrefix, baseDir, httpPrefix,
self.server.translate, self.server.translate,
nickname, domain, port, nickname, domain, port,
maxPostsInRSSFeed, 1) maxPostsInRSSFeed, 1,
systemLanguage)
if msg is not None: if msg is not None:
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -6233,7 +6242,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.personCache, self.server.personCache,
httpPrefix, httpPrefix,
self.server.projectVersion, self.server.projectVersion,
self.server.YTReplacementDomain) self.server.YTReplacementDomain,
self.server.systemLanguage)
if hashtagStr: if hashtagStr:
msg = hashtagStr.encode('utf-8') msg = hashtagStr.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -9767,7 +9777,8 @@ class PubServer(BaseHTTPRequestHandler):
nickname, nickname,
domain, port, domain, port,
maxPostsInBlogsFeed, pageNumber, maxPostsInBlogsFeed, pageNumber,
self.server.peertubeInstances) self.server.peertubeInstances,
self.server.systemLanguage)
if msg is not None: if msg is not None:
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -10384,7 +10395,8 @@ class PubServer(BaseHTTPRequestHandler):
translate, baseDir, translate, baseDir,
path, domain, port, path, domain, port,
httpPrefix, httpPrefix,
postUrl).encode('utf-8') postUrl,
self.server.systemLanguage).encode('utf-8')
if msg: if msg:
msglen = len(msg) msglen = len(msg)
self._set_headers('text/html', msglen, self._set_headers('text/html', msglen,
@ -10732,7 +10744,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.port, self.server.port,
self.server.proxyType, self.server.proxyType,
GETstartTime, GETtimings, GETstartTime, GETtimings,
self.server.debug) self.server.debug,
self.server.systemLanguage)
return return
usersInPath = False usersInPath = False
@ -10894,7 +10907,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain, self.server.domain,
self.server.port, self.server.port,
maxPostsInBlogsFeed, maxPostsInBlogsFeed,
self.server.peertubeInstances) self.server.peertubeInstances,
self.server.systemLanguage)
if msg is not None: if msg is not None:
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -12265,7 +12279,7 @@ class PubServer(BaseHTTPRequestHandler):
self.path, self.path,
replyPageNumber, replyPageNumber,
nickname, self.server.domain, nickname, self.server.domain,
postUrl) postUrl, self.server.systemLanguage)
if msg: if msg:
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
@ -12643,7 +12657,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain, self.server.domain,
self.server.port, self.server.port,
searchHandle, searchHandle,
self.server.debug) self.server.debug,
self.server.systemLanguage)
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
self._login_headers('text/html', self._login_headers('text/html',
@ -12677,7 +12692,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain, self.server.domain,
self.server.port, self.server.port,
searchHandle, searchHandle,
self.server.debug) self.server.debug,
self.server.systemLanguage)
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
msglen = len(msg) msglen = len(msg)
self._login_headers('text/html', self._login_headers('text/html',
@ -13122,9 +13138,11 @@ class PubServer(BaseHTTPRequestHandler):
if fields['schedulePost']: if fields['schedulePost']:
return 1 return 1
if pinToProfile: if pinToProfile:
contentStr = \
getContentFromPost(messageJson,
self.server.systemLanguage)
pinPost(self.server.baseDir, pinPost(self.server.baseDir,
nickname, self.server.domain, nickname, self.server.domain, contentStr)
messageJson['object']['content'])
return 1 return 1
if self._postToOutbox(messageJson, __version__, nickname): if self._postToOutbox(messageJson, __version__, nickname):
populateReplies(self.server.baseDir, populateReplies(self.server.baseDir,
@ -13251,6 +13269,9 @@ class PubServer(BaseHTTPRequestHandler):
tags, 'content') tags, 'content')
postJsonObject['object']['content'] = fields['message'] postJsonObject['object']['content'] = fields['message']
contentMap = postJsonObject['object']['contentMap']
contentMap[self.server.systemLanguage] = \
fields['message']
imgDescription = '' imgDescription = ''
if fields.get('imageDescription'): if fields.get('imageDescription'):
@ -13274,7 +13295,8 @@ class PubServer(BaseHTTPRequestHandler):
city) city)
replaceYouTube(postJsonObject, replaceYouTube(postJsonObject,
self.server.YTReplacementDomain) self.server.YTReplacementDomain,
self.server.systemLanguage)
saveJson(postJsonObject, postFilename) saveJson(postJsonObject, postFilename)
# also save to the news actor # also save to the news actor
if nickname != 'news': if nickname != 'news':

25
desktop_client.py
View File

@ -16,6 +16,7 @@ import webbrowser
import urllib.parse import urllib.parse
from pathlib import Path from pathlib import Path
from random import randint from random import randint
from utils import getContentFromPost
from utils import hasObjectDict from utils import hasObjectDict
from utils import getFullDomain from utils import getFullDomain
from utils import isDM from utils import isDM
@ -689,15 +690,16 @@ def _readLocalBoxPost(session, nickname: str, domain: str,
__version__, translate, __version__, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, False) recentPostsCache, False,
systemLanguage)
if postJsonObject2: if postJsonObject2:
if hasObjectDict(postJsonObject2): if hasObjectDict(postJsonObject2):
if postJsonObject2['object'].get('attributedTo') and \ if postJsonObject2['object'].get('attributedTo') and \
postJsonObject2['object'].get('content'): postJsonObject2['object'].get('content'):
attributedTo = postJsonObject2['object']['attributedTo'] attributedTo = postJsonObject2['object']['attributedTo']
content = postJsonObject2['object']['content'] content = \
if isinstance(attributedTo, str) and \ getContentFromPost(postJsonObject2, systemLanguage)
isinstance(content, str): if isinstance(attributedTo, str) and content:
actor = attributedTo actor = attributedTo
nameStr += ' ' + translate['announces'] + ' ' + \ nameStr += ' ' + translate['announces'] + ' ' + \
getNicknameFromActor(actor) getNicknameFromActor(actor)
@ -721,7 +723,7 @@ def _readLocalBoxPost(session, nickname: str, domain: str,
attributedTo = postJsonObject['object']['attributedTo'] attributedTo = postJsonObject['object']['attributedTo']
if not attributedTo: if not attributedTo:
return {} return {}
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
if not isinstance(attributedTo, str) or \ if not isinstance(attributedTo, str) or \
not isinstance(content, str): not isinstance(content, str):
return {} return {}
@ -1044,7 +1046,8 @@ def _desktopShowBox(indent: str,
published = _formatPublished(postJsonObject['published']) published = _formatPublished(postJsonObject['published'])
content = _textOnlyContent(postJsonObject['object']['content']) contentStr = getContentFromPost(postJsonObject, systemLanguage)
content = _textOnlyContent(contentStr)
if boxName != 'dm': if boxName != 'dm':
if isDM(postJsonObject): if isDM(postJsonObject):
content = '📧' + content content = '📧' + content
@ -2321,11 +2324,13 @@ def runDesktopClient(baseDir: str, proxyType: str, httpPrefix: str,
__version__, translate, __version__, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, False) recentPostsCache, False,
systemLanguage)
if postJsonObject2: if postJsonObject2:
postJsonObject = postJsonObject2 postJsonObject = postJsonObject2
if postJsonObject: if postJsonObject:
content = postJsonObject['object']['content'] content = \
getContentFromPost(postJsonObject, systemLanguage)
messageStr, detectedLinks = \ messageStr, detectedLinks = \
speakableText(baseDir, content, translate) speakableText(baseDir, content, translate)
linkOpened = False linkOpened = False
@ -2381,7 +2386,9 @@ def runDesktopClient(baseDir: str, proxyType: str, httpPrefix: str,
print('') print('')
if postJsonObject['object'].get('summary'): if postJsonObject['object'].get('summary'):
print(postJsonObject['object']['summary']) print(postJsonObject['object']['summary'])
print(postJsonObject['object']['content']) contentStr = getContentFromPost(postJsonObject,
systemLanguage)
print(contentStr)
print('') print('')
sayStr = 'Confirm delete, yes or no?' sayStr = 'Confirm delete, yes or no?'
_sayCommand(sayStr, sayStr, screenreader, _sayCommand(sayStr, sayStr, screenreader,

26
epicyon.py
View File

@ -631,9 +631,11 @@ if args.posts:
args.port = 80 args.port = 80
elif args.gnunet: elif args.gnunet:
proxyType = 'gnunet' proxyType = 'gnunet'
if not args.language:
args.language = 'en'
getPublicPostsOfPerson(baseDir, nickname, domain, False, True, getPublicPostsOfPerson(baseDir, nickname, domain, False, True,
proxyType, args.port, httpPrefix, debug, proxyType, args.port, httpPrefix, debug,
__version__) __version__, args.language)
sys.exit() sys.exit()
if args.postDomains: if args.postDomains:
@ -663,12 +665,15 @@ if args.postDomains:
proxyType = 'gnunet' proxyType = 'gnunet'
wordFrequency = {} wordFrequency = {}
domainList = [] domainList = []
if not args.language:
args.language = 'en'
domainList = getPublicPostDomains(None, domainList = getPublicPostDomains(None,
baseDir, nickname, domain, baseDir, nickname, domain,
proxyType, args.port, proxyType, args.port,
httpPrefix, debug, httpPrefix, debug,
__version__, __version__,
wordFrequency, domainList) wordFrequency, domainList,
args.language)
for postDomain in domainList: for postDomain in domainList:
print(postDomain) print(postDomain)
sys.exit() sys.exit()
@ -703,12 +708,15 @@ if args.postDomainsBlocked:
proxyType = 'gnunet' proxyType = 'gnunet'
wordFrequency = {} wordFrequency = {}
domainList = [] domainList = []
if not args.language:
args.language = 'en'
domainList = getPublicPostDomainsBlocked(None, domainList = getPublicPostDomainsBlocked(None,
baseDir, nickname, domain, baseDir, nickname, domain,
proxyType, args.port, proxyType, args.port,
httpPrefix, debug, httpPrefix, debug,
__version__, __version__,
wordFrequency, domainList) wordFrequency, domainList,
args.language)
for postDomain in domainList: for postDomain in domainList:
print(postDomain) print(postDomain)
sys.exit() sys.exit()
@ -741,12 +749,14 @@ if args.checkDomains:
elif args.gnunet: elif args.gnunet:
proxyType = 'gnunet' proxyType = 'gnunet'
maxBlockedDomains = 0 maxBlockedDomains = 0
if not args.language:
args.language = 'en'
checkDomains(None, checkDomains(None,
baseDir, nickname, domain, baseDir, nickname, domain,
proxyType, args.port, proxyType, args.port,
httpPrefix, debug, httpPrefix, debug,
__version__, __version__,
maxBlockedDomains, False) maxBlockedDomains, False, args.language)
sys.exit() sys.exit()
if args.socnet: if args.socnet:
@ -758,10 +768,12 @@ if args.socnet:
if not args.http: if not args.http:
args.port = 443 args.port = 443
proxyType = 'tor' proxyType = 'tor'
if not args.language:
args.language = 'en'
dotGraph = instancesGraph(baseDir, args.socnet, dotGraph = instancesGraph(baseDir, args.socnet,
proxyType, args.port, proxyType, args.port,
httpPrefix, debug, httpPrefix, debug,
__version__) __version__, args.language)
try: try:
with open('socnet.dot', 'w+') as fp: with open('socnet.dot', 'w+') as fp:
fp.write(dotGraph) fp.write(dotGraph)
@ -785,9 +797,11 @@ if args.postsraw:
proxyType = 'i2p' proxyType = 'i2p'
elif args.gnunet: elif args.gnunet:
proxyType = 'gnunet' proxyType = 'gnunet'
if not args.language:
args.language = 'en'
getPublicPostsOfPerson(baseDir, nickname, domain, False, False, getPublicPostsOfPerson(baseDir, nickname, domain, False, False,
proxyType, args.port, httpPrefix, debug, proxyType, args.port, httpPrefix, debug,
__version__) __version__, args.language)
sys.exit() sys.exit()
if args.json: if args.json:

3
happening.py
View File

@ -55,6 +55,9 @@ def saveEventPost(baseDir: str, handle: str, postId: str,
See https://framagit.org/framasoft/mobilizon/-/blob/ See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex master/lib/federation/activity_stream/converter/event.ex
""" """
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('WARN: Account does not exist at ' +
baseDir + '/accounts/' + handle)
calendarPath = baseDir + '/accounts/' + handle + '/calendar' calendarPath = baseDir + '/accounts/' + handle + '/calendar'
if not os.path.isdir(calendarPath): if not os.path.isdir(calendarPath):
os.mkdir(calendarPath) os.mkdir(calendarPath)

92
inbox.py
View File

@ -13,6 +13,7 @@ import datetime
import time import time
import random import random
from linked_data_sig import verifyJsonSignature from linked_data_sig import verifyJsonSignature
from utils import getContentFromPost
from utils import acctDir from utils import acctDir
from utils import removeDomainPort from utils import removeDomainPort
from utils import getPortFromDomain from utils import getPortFromDomain
@ -23,7 +24,6 @@ from utils import getConfigParam
from utils import hasUsersPath from utils import hasUsersPath
from utils import validPostDate from utils import validPostDate
from utils import getFullDomain from utils import getFullDomain
from utils import isEventPost
from utils import removeIdEnding from utils import removeIdEnding
from utils import getProtocolPrefixes from utils import getProtocolPrefixes
from utils import isBlogPost from utils import isBlogPost
@ -351,7 +351,7 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str,
messageBytes: str, messageBytes: str,
httpHeaders: {}, httpHeaders: {},
postPath: str, debug: bool, postPath: str, debug: bool,
blockedCache: []) -> str: blockedCache: [], systemLanguage: str) -> str:
"""Saves the give json to the inbox queue for the person """Saves the give json to the inbox queue for the person
keyId specifies the actor sending the post keyId specifies the actor sending the post
""" """
@ -415,9 +415,9 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str,
replyNickname + '@' + replyDomain) replyNickname + '@' + replyDomain)
return None return None
if postJsonObject['object'].get('content'): if postJsonObject['object'].get('content'):
if isinstance(postJsonObject['object']['content'], str): contentStr = getContentFromPost(postJsonObject, systemLanguage)
if isFiltered(baseDir, nickname, domain, if contentStr:
postJsonObject['object']['content']): if isFiltered(baseDir, nickname, domain, contentStr):
if debug: if debug:
print('WARN: post was filtered out due to content') print('WARN: post was filtered out due to content')
return None return None
@ -730,25 +730,6 @@ def _receiveUndo(session, baseDir: str, httpPrefix: str,
return False return False
def _receiveEventPost(recentPostsCache: {}, session, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
nickname: str, debug: bool) -> bool:
"""Receive a mobilizon-type event activity
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
"""
if not isEventPost(messageJson):
return
print('Receiving event: ' + str(messageJson['object']))
handle = getFullDomain(nickname + '@' + domain, port)
postId = removeIdEnding(messageJson['id']).replace('/', '#')
saveEventPost(baseDir, handle, postId, messageJson['object'])
def _personReceiveUpdate(baseDir: str, def _personReceiveUpdate(baseDir: str,
domain: str, port: int, domain: str, port: int,
updateNickname: str, updateDomain: str, updateNickname: str, updateDomain: str,
@ -1305,7 +1286,7 @@ def _receiveAnnounce(recentPostsCache: {},
debug: bool, translate: {}, debug: bool, translate: {},
YTReplacementDomain: str, YTReplacementDomain: str,
allowLocalNetworkAccess: bool, allowLocalNetworkAccess: bool,
themeName: str) -> bool: themeName: str, systemLanguage: str) -> bool:
"""Receives an announce activity within the POST section of HTTPServer """Receives an announce activity within the POST section of HTTPServer
""" """
if messageJson['type'] != 'Announce': if messageJson['type'] != 'Announce':
@ -1392,7 +1373,8 @@ def _receiveAnnounce(recentPostsCache: {},
__version__, translate, __version__, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, debug) recentPostsCache, debug,
systemLanguage)
if not postJsonObject: if not postJsonObject:
notInOnion = True notInOnion = True
if onionDomain: if onionDomain:
@ -1616,7 +1598,8 @@ def _estimateNumberOfEmoji(content: str) -> int:
def _validPostContent(baseDir: str, nickname: str, domain: str, def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson: {}, maxMentions: int, maxEmoji: int, messageJson: {}, maxMentions: int, maxEmoji: int,
allowLocalNetworkAccess: bool, debug: bool) -> bool: allowLocalNetworkAccess: bool, debug: bool,
systemLanguage: str) -> bool:
"""Is the content of a received post valid? """Is the content of a received post valid?
Check for bad html Check for bad html
Check for hellthreads Check for hellthreads
@ -1651,27 +1634,27 @@ def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson['object']['content']): messageJson['object']['content']):
return True return True
if dangerousMarkup(messageJson['object']['content'], contentStr = getContentFromPost(messageJson, systemLanguage)
allowLocalNetworkAccess): if dangerousMarkup(contentStr, allowLocalNetworkAccess):
if messageJson['object'].get('id'): if messageJson['object'].get('id'):
print('REJECT ARBITRARY HTML: ' + messageJson['object']['id']) print('REJECT ARBITRARY HTML: ' + messageJson['object']['id'])
print('REJECT ARBITRARY HTML: bad string in post - ' + print('REJECT ARBITRARY HTML: bad string in post - ' +
messageJson['object']['content']) contentStr)
return False return False
# check (rough) number of mentions # check (rough) number of mentions
mentionsEst = _estimateNumberOfMentions(messageJson['object']['content']) mentionsEst = _estimateNumberOfMentions(contentStr)
if mentionsEst > maxMentions: if mentionsEst > maxMentions:
if messageJson['object'].get('id'): if messageJson['object'].get('id'):
print('REJECT HELLTHREAD: ' + messageJson['object']['id']) print('REJECT HELLTHREAD: ' + messageJson['object']['id'])
print('REJECT HELLTHREAD: Too many mentions in post - ' + print('REJECT HELLTHREAD: Too many mentions in post - ' +
messageJson['object']['content']) contentStr)
return False return False
if _estimateNumberOfEmoji(messageJson['object']['content']) > maxEmoji: if _estimateNumberOfEmoji(contentStr) > maxEmoji:
if messageJson['object'].get('id'): if messageJson['object'].get('id'):
print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id']) print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id'])
print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' + print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' +
messageJson['object']['content']) contentStr)
return False return False
# check number of tags # check number of tags
if messageJson['object'].get('tag'): if messageJson['object'].get('tag'):
@ -1685,8 +1668,7 @@ def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson['object']['tag']) messageJson['object']['tag'])
return False return False
# check for filtered content # check for filtered content
if isFiltered(baseDir, nickname, domain, if isFiltered(baseDir, nickname, domain, contentStr):
messageJson['object']['content']):
print('REJECT: content filtered') print('REJECT: content filtered')
return False return False
if messageJson['object'].get('inReplyTo'): if messageJson['object'].get('inReplyTo'):
@ -1926,7 +1908,8 @@ def _sendToGroupMembers(session, baseDir: str, handle: str, port: int,
postJsonObject: {}, postJsonObject: {},
httpPrefix: str, federationList: [], httpPrefix: str, federationList: [],
sendThreads: [], postLog: [], cachedWebfingers: {}, sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, debug: bool) -> None: personCache: {}, debug: bool,
systemLanguage: str) -> None:
"""When a post arrives for a group send it out to the group members """When a post arrives for a group send it out to the group members
""" """
followersFile = baseDir + '/accounts/' + handle + '/followers.txt' followersFile = baseDir + '/accounts/' + handle + '/followers.txt'
@ -1947,9 +1930,12 @@ def _sendToGroupMembers(session, baseDir: str, handle: str, port: int,
sendingActorDomainFull = \ sendingActorDomainFull = \
getFullDomain(sendingActorDomain, sendingActorPort) getFullDomain(sendingActorDomain, sendingActorPort)
senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull
if not postJsonObject['object']['content'].startswith(senderStr): contentStr = getContentFromPost(postJsonObject, systemLanguage)
if not contentStr.startswith(senderStr):
postJsonObject['object']['content'] = \ postJsonObject['object']['content'] = \
senderStr + ' ' + postJsonObject['object']['content'] senderStr + ' ' + contentStr
postJsonObject['object']['contentMap'][systemLanguage] = \
senderStr + ' ' + contentStr
# add mention to tag list # add mention to tag list
if not postJsonObject['object']['tag']: if not postJsonObject['object']['tag']:
postJsonObject['object']['tag'] = [] postJsonObject['object']['tag'] = []
@ -2367,7 +2353,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
debug, translate, debug, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
themeName): themeName, systemLanguage):
if debug: if debug:
print('DEBUG: Announce accepted from ' + actor) print('DEBUG: Announce accepted from ' + actor)
@ -2416,7 +2402,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
jsonObj = None jsonObj = None
if _validPostContent(baseDir, nickname, domain, if _validPostContent(baseDir, nickname, domain,
postJsonObject, maxMentions, maxEmoji, postJsonObject, maxMentions, maxEmoji,
allowLocalNetworkAccess, debug): allowLocalNetworkAccess, debug,
systemLanguage):
if postJsonObject.get('object'): if postJsonObject.get('object'):
jsonObj = postJsonObject['object'] jsonObj = postJsonObject['object']
@ -2449,7 +2436,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
return False return False
# replace YouTube links, so they get less tracking data # replace YouTube links, so they get less tracking data
replaceYouTube(postJsonObject, YTReplacementDomain) replaceYouTube(postJsonObject, YTReplacementDomain, systemLanguage)
# list of indexes to be updated # list of indexes to be updated
updateIndexList = ['inbox'] updateIndexList = ['inbox']
@ -2519,7 +2506,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
nickname, domain, postJsonObject, nickname, domain, postJsonObject,
translate, YTReplacementDomain, translate, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, debug): recentPostsCache, debug, systemLanguage):
# media index will be updated # media index will be updated
updateIndexList.append('tlmedia') updateIndexList.append('tlmedia')
if isBlogPost(postJsonObject): if isBlogPost(postJsonObject):
@ -2613,7 +2600,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
postJsonObject, postJsonObject,
httpPrefix, federationList, sendThreads, httpPrefix, federationList, sendThreads,
postLog, cachedWebfingers, personCache, postLog, cachedWebfingers, personCache,
debug) debug, systemLanguage)
# if the post wasn't saved # if the post wasn't saved
if not os.path.isfile(destinationFilename): if not os.path.isfile(destinationFilename):
@ -3149,23 +3136,6 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
queue.pop(0) queue.pop(0)
continue continue
if _receiveEventPost(recentPostsCache, session,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
queueJson['postNickname'],
debug):
print('Queue: Event activity accepted from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
if _receiveUpdate(recentPostsCache, session, if _receiveUpdate(recentPostsCache, session,
baseDir, httpPrefix, baseDir, httpPrefix,
domain, port, domain, port,

13
media.py
View File

@ -13,6 +13,7 @@ import subprocess
from random import randint from random import randint
from hashlib import sha1 from hashlib import sha1
from auth import createPassword from auth import createPassword
from utils import getContentFromPost
from utils import getFullDomain from utils import getFullDomain
from utils import getImageExtensions from utils import getImageExtensions
from utils import getVideoExtensions from utils import getVideoExtensions
@ -26,7 +27,8 @@ from shutil import move
from city import spoofGeolocation from city import spoofGeolocation
def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None: def replaceYouTube(postJsonObject: {}, replacementDomain: str,
systemLanguage: str) -> None:
"""Replace YouTube with a replacement domain """Replace YouTube with a replacement domain
This denies Google some, but not all, tracking data This denies Google some, but not all, tracking data
""" """
@ -36,11 +38,12 @@ def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None:
return return
if not postJsonObject['object'].get('content'): if not postJsonObject['object'].get('content'):
return return
if 'www.youtube.com' not in postJsonObject['object']['content']: contentStr = getContentFromPost(postJsonObject, systemLanguage)
if 'www.youtube.com' not in contentStr:
return return
postJsonObject['object']['content'] = \ contentStr = contentStr.replace('www.youtube.com', replacementDomain)
postJsonObject['object']['content'].replace('www.youtube.com', postJsonObject['object']['content'] = contentStr
replacementDomain) postJsonObject['object']['contentMap'][systemLanguage] = contentStr
def _removeMetaData(imageFilename: str, outputFilename: str) -> None: def _removeMetaData(imageFilename: str, outputFilename: str) -> None:

29
newsdaemon.py
View File

@ -25,6 +25,7 @@ from newswire import getDictFromNewswire
from posts import createNewsPost from posts import createNewsPost
from posts import archivePostsForPerson from posts import archivePostsForPerson
from content import validHashTag from content import validHashTag
from utils import getContentFromPost
from utils import removeHtml from utils import removeHtml
from utils import getFullDomain from utils import getFullDomain
from utils import loadJson from utils import loadJson
@ -279,7 +280,7 @@ def hashtagRuleTree(operators: [],
def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str, def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
postJsonObject: {}, postJsonObject: {},
actionStr: str, hashtags: []) -> None: actionStr: str, hashtags: [], systemLanguage: str) -> None:
"""Adds a hashtag via a hashtag rule """Adds a hashtag via a hashtag rule
""" """
addHashtag = actionStr.split('add ', 1)[1].strip() addHashtag = actionStr.split('add ', 1)[1].strip()
@ -313,7 +314,7 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
hashtagHtml = \ hashtagHtml = \
" <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \ " <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>" "rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
if hashtagHtml in content: if hashtagHtml in content:
return return
@ -328,7 +329,7 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {}, def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
actionStr: str, hashtags: []) -> None: actionStr: str, hashtags: [], systemLanguage: str) -> None:
"""Removes a hashtag via a hashtag rule """Removes a hashtag via a hashtag rule
""" """
rmHashtag = actionStr.split('remove ', 1)[1].strip() rmHashtag = actionStr.split('remove ', 1)[1].strip()
@ -343,10 +344,11 @@ def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
hashtagHtml = \ hashtagHtml = \
"<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \ "<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>" "rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content'] content = getContentFromPost(postJsonObject, systemLanguage)
if hashtagHtml in content: if hashtagHtml in content:
content = content.replace(hashtagHtml, '').replace(' ', ' ') content = content.replace(hashtagHtml, '').replace(' ', ' ')
postJsonObject['object']['content'] = content postJsonObject['object']['content'] = content
postJsonObject['object']['contentMap'][systemLanguage] = content
rmTagObject = None rmTagObject = None
for t in postJsonObject['object']['tag']: for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'): if t.get('type') and t.get('name'):
@ -365,7 +367,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
cachedWebfingers: {}, cachedWebfingers: {},
federationList: [], federationList: [],
sendThreads: [], postLog: [], sendThreads: [], postLog: [],
moderated: bool, url: str) -> bool: moderated: bool, url: str,
systemLanguage: str) -> bool:
"""Applies hashtag rules to a news post. """Applies hashtag rules to a news post.
Returns true if the post should be saved to the news timeline Returns true if the post should be saved to the news timeline
of this instance of this instance
@ -382,7 +385,7 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
# get the full text content of the post # get the full text content of the post
content = '' content = ''
if postJsonObject['object'].get('content'): if postJsonObject['object'].get('content'):
content += postJsonObject['object']['content'] content += getContentFromPost(postJsonObject, systemLanguage)
if postJsonObject['object'].get('summary'): if postJsonObject['object'].get('summary'):
content += ' ' + postJsonObject['object']['summary'] content += ' ' + postJsonObject['object']['summary']
content = content.lower() content = content.lower()
@ -409,11 +412,11 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
if actionStr.startswith('add '): if actionStr.startswith('add '):
# add a hashtag # add a hashtag
_hashtagAdd(baseDir, httpPrefix, domainFull, _hashtagAdd(baseDir, httpPrefix, domainFull,
postJsonObject, actionStr, hashtags) postJsonObject, actionStr, hashtags, systemLanguage)
elif actionStr.startswith('remove '): elif actionStr.startswith('remove '):
# remove a hashtag # remove a hashtag
_hashtagRemove(httpPrefix, domainFull, postJsonObject, _hashtagRemove(httpPrefix, domainFull, postJsonObject,
actionStr, hashtags) actionStr, hashtags, systemLanguage)
elif actionStr.startswith('block') or actionStr.startswith('drop'): elif actionStr.startswith('block') or actionStr.startswith('drop'):
# Block this item # Block this item
return False return False
@ -627,7 +630,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
blog['object']['published'] = dateStr blog['object']['published'] = dateStr
blog['object']['content'] = rssDescription blog['object']['content'] = rssDescription
blog['object']['contentMap']['en'] = rssDescription blog['object']['contentMap'][systemLanguage] = rssDescription
domainFull = getFullDomain(domain, port) domainFull = getFullDomain(domain, port)
@ -642,7 +645,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
personCache, cachedWebfingers, personCache, cachedWebfingers,
federationList, federationList,
sendThreads, postLog, sendThreads, postLog,
moderated, url) moderated, url, systemLanguage)
# save the post and update the index # save the post and update the index
if savePost: if savePost:
@ -664,7 +667,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
"\" class=\"addedHashtag\" " + \ "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + \ "rel=\"tag\">#<span>" + \
htId + "</span></a>" htId + "</span></a>"
content = blog['object']['content'] content = getContentFromPost(blog, systemLanguage)
if hashtagHtml not in content: if hashtagHtml not in content:
if content.endswith('</p>'): if content.endswith('</p>'):
content = \ content = \
@ -673,6 +676,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
else: else:
content += hashtagHtml content += hashtagHtml
blog['object']['content'] = content blog['object']['content'] = content
blog['object']['contentMap'][systemLanguage] = content
# update the newswire tags if new ones have been found by # update the newswire tags if new ones have been found by
# _newswireHashtagProcessing # _newswireHashtagProcessing
@ -749,7 +753,8 @@ def runNewswireDaemon(baseDir: str, httpd,
httpd.maxTags, httpd.maxTags,
httpd.maxFeedItemSizeKb, httpd.maxFeedItemSizeKb,
httpd.maxNewswirePosts, httpd.maxNewswirePosts,
httpd.maxCategoriesFeedItemSizeKb) httpd.maxCategoriesFeedItemSizeKb,
httpd.systemLanguage)
if not httpd.newswire: if not httpd.newswire:
if os.path.isfile(newswireStateFilename): if os.path.isfile(newswireStateFilename):

16
newswire.py
View File

@ -18,6 +18,7 @@ from datetime import timezone
from collections import OrderedDict from collections import OrderedDict
from utils import validPostDate from utils import validPostDate
from categories import setHashtagCategory from categories import setHashtagCategory
from utils import getContentFromPost
from utils import hasObjectDict from utils import hasObjectDict
from utils import firstParagraphFromString from utils import firstParagraphFromString
from utils import isPublicPost from utils import isPublicPost
@ -909,7 +910,7 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
newswire: {}, newswire: {},
maxBlogsPerAccount: int, maxBlogsPerAccount: int,
indexFilename: str, indexFilename: str,
maxTags: int) -> None: maxTags: int, systemLanguage: str) -> None:
"""Adds blogs for the given account to the newswire """Adds blogs for the given account to the newswire
""" """
if not os.path.isfile(indexFilename): if not os.path.isfile(indexFilename):
@ -961,7 +962,8 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
votes = [] votes = []
if os.path.isfile(fullPostFilename + '.votes'): if os.path.isfile(fullPostFilename + '.votes'):
votes = loadJson(fullPostFilename + '.votes') votes = loadJson(fullPostFilename + '.votes')
content = postJsonObject['object']['content'] content = \
getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content) description = firstParagraphFromString(content)
description = removeHtml(description) description = removeHtml(description)
tagsFromPost = _getHashtagsFromPost(postJsonObject) tagsFromPost = _getHashtagsFromPost(postJsonObject)
@ -981,7 +983,7 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {}, def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
maxBlogsPerAccount: int, maxBlogsPerAccount: int,
maxTags: int) -> None: maxTags: int, systemLanguage: str) -> None:
"""Adds blogs from each user account into the newswire """Adds blogs from each user account into the newswire
""" """
moderationDict = {} moderationDict = {}
@ -1009,7 +1011,8 @@ def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
domain = handle.split('@')[1] domain = handle.split('@')[1]
_addAccountBlogsToNewswire(baseDir, nickname, domain, _addAccountBlogsToNewswire(baseDir, nickname, domain,
newswire, maxBlogsPerAccount, newswire, maxBlogsPerAccount,
blogsIndex, maxTags) blogsIndex, maxTags,
systemLanguage)
break break
# sort the moderation dict into chronological order, latest first # sort the moderation dict into chronological order, latest first
@ -1029,7 +1032,8 @@ def getDictFromNewswire(session, baseDir: str, domain: str,
maxPostsPerSource: int, maxFeedSizeKb: int, maxPostsPerSource: int, maxFeedSizeKb: int,
maxTags: int, maxFeedItemSizeKb: int, maxTags: int, maxFeedItemSizeKb: int,
maxNewswirePosts: int, maxNewswirePosts: int,
maxCategoriesFeedItemSizeKb: int) -> {}: maxCategoriesFeedItemSizeKb: int,
systemLanguage: str) -> {}:
"""Gets rss feeds as a dictionary from newswire file """Gets rss feeds as a dictionary from newswire file
""" """
subscriptionsFilename = baseDir + '/accounts/newswire.txt' subscriptionsFilename = baseDir + '/accounts/newswire.txt'
@ -1077,7 +1081,7 @@ def getDictFromNewswire(session, baseDir: str, domain: str,
# add blogs from each user account # add blogs from each user account
_addBlogsToNewswire(baseDir, domain, result, _addBlogsToNewswire(baseDir, domain, result,
maxPostsPerSource, maxTags) maxPostsPerSource, maxTags, systemLanguage)
# sort into chronological order, latest first # sort into chronological order, latest first
sortedResult = OrderedDict(sorted(result.items(), reverse=True)) sortedResult = OrderedDict(sorted(result.items(), reverse=True))

13
outbox.py
View File

@ -16,6 +16,7 @@ from posts import outboxMessageCreateWrap
from posts import savePostToBox from posts import savePostToBox
from posts import sendToFollowersThread from posts import sendToFollowersThread
from posts import sendToNamedAddresses from posts import sendToNamedAddresses
from utils import getContentFromPost
from utils import hasObjectDict from utils import hasObjectDict
from utils import getLocalNetworkAddresses from utils import getLocalNetworkAddresses
from utils import getFullDomain from utils import getFullDomain
@ -178,7 +179,7 @@ def postMessageToOutbox(session, translate: {},
YTReplacementDomain: str, YTReplacementDomain: str,
showPublishedDateOnly: bool, showPublishedDateOnly: bool,
allowLocalNetworkAccess: bool, allowLocalNetworkAccess: bool,
city: str) -> bool: city: str, systemLanguage: str) -> bool:
"""post is received by the outbox """post is received by the outbox
Client to server message post Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
@ -201,9 +202,9 @@ def postMessageToOutbox(session, translate: {},
# check that the outgoing post doesn't contain any markup # check that the outgoing post doesn't contain any markup
# which can be used to implement exploits # which can be used to implement exploits
if hasObjectDict(messageJson): if hasObjectDict(messageJson):
if messageJson['object'].get('content'): contentStr = getContentFromPost(messageJson, systemLanguage)
if dangerousMarkup(messageJson['object']['content'], if contentStr:
allowLocalNetworkAccess): if dangerousMarkup(contentStr, allowLocalNetworkAccess):
print('POST to outbox contains dangerous markup: ' + print('POST to outbox contains dangerous markup: ' +
str(messageJson)) str(messageJson))
return False return False
@ -264,7 +265,7 @@ def postMessageToOutbox(session, translate: {},
print('DEBUG: domain is blocked: ' + messageJson['actor']) print('DEBUG: domain is blocked: ' + messageJson['actor'])
return False return False
# replace youtube, so that google gets less tracking data # replace youtube, so that google gets less tracking data
replaceYouTube(messageJson, YTReplacementDomain) replaceYouTube(messageJson, YTReplacementDomain, systemLanguage)
# https://www.w3.org/TR/activitypub/#create-activity-outbox # https://www.w3.org/TR/activitypub/#create-activity-outbox
messageJson['object']['attributedTo'] = messageJson['actor'] messageJson['object']['attributedTo'] = messageJson['actor']
if messageJson['object'].get('attachment'): if messageJson['object'].get('attachment'):
@ -390,7 +391,7 @@ def postMessageToOutbox(session, translate: {},
messageJson, messageJson,
translate, YTReplacementDomain, translate, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, debug): recentPostsCache, debug, systemLanguage):
inboxUpdateIndex('tlmedia', baseDir, inboxUpdateIndex('tlmedia', baseDir,
postToNickname + '@' + domain, postToNickname + '@' + domain,
savedFilename, debug) savedFilename, debug)

54
posts.py
View File

@ -31,6 +31,7 @@ from session import postImage
from webfinger import webfingerHandle from webfinger import webfingerHandle
from httpsig import createSignedHeader from httpsig import createSignedHeader
from siteactive import siteIsActive from siteactive import siteIsActive
from utils import getContentFromPost
from utils import removeDomainPort from utils import removeDomainPort
from utils import getPortFromDomain from utils import getPortFromDomain
from utils import hasObjectDict from utils import hasObjectDict
@ -322,7 +323,7 @@ def _getPosts(session, outboxUrl: str, maxPosts: int,
personCache: {}, raw: bool, personCache: {}, raw: bool,
simple: bool, debug: bool, simple: bool, debug: bool,
projectVersion: str, httpPrefix: str, projectVersion: str, httpPrefix: str,
domain: str) -> {}: domain: str, systemLanguage: str) -> {}:
"""Gets public posts from an outbox """Gets public posts from an outbox
""" """
personPosts = {} personPosts = {}
@ -385,8 +386,8 @@ def _getPosts(session, outboxUrl: str, maxPosts: int,
if not isPublic: if not isPublic:
continue continue
content = \ content = getContentFromPost(item, systemLanguage)
item['object']['content'].replace('&apos;', "'") content = content.replace('&apos;', "'")
mentions = [] mentions = []
emoji = {} emoji = {}
@ -538,7 +539,7 @@ def getPostDomains(session, outboxUrl: str, maxPosts: int,
projectVersion: str, httpPrefix: str, projectVersion: str, httpPrefix: str,
domain: str, domain: str,
wordFrequency: {}, wordFrequency: {},
domainList=[]) -> []: domainList: [], systemLanguage: str) -> []:
"""Returns a list of domains referenced within public posts """Returns a list of domains referenced within public posts
""" """
if not outboxUrl: if not outboxUrl:
@ -563,9 +564,9 @@ def getPostDomains(session, outboxUrl: str, maxPosts: int,
break break
if not hasObjectDict(item): if not hasObjectDict(item):
continue continue
if item['object'].get('content'): contentStr = getContentFromPost(item, systemLanguage)
_updateWordFrequency(item['object']['content'], if contentStr:
wordFrequency) _updateWordFrequency(contentStr, wordFrequency)
if item['object'].get('inReplyTo'): if item['object'].get('inReplyTo'):
if isinstance(item['object']['inReplyTo'], str): if isinstance(item['object']['inReplyTo'], str):
postDomain, postPort = \ postDomain, postPort = \
@ -2932,7 +2933,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str,
postJsonObject: {}, translate: {}, postJsonObject: {}, translate: {},
YTReplacementDomain: str, YTReplacementDomain: str,
allowLocalNetworkAccess: bool, allowLocalNetworkAccess: bool,
recentPostsCache: {}, debug: bool) -> bool: recentPostsCache: {}, debug: bool,
systemLanguage: str) -> bool:
"""Returns true if the given post has attached image media """Returns true if the given post has attached image media
""" """
if postJsonObject['type'] == 'Announce': if postJsonObject['type'] == 'Announce':
@ -2942,7 +2944,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str,
__version__, translate, __version__, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, debug) recentPostsCache, debug,
systemLanguage)
if postJsonAnnounce: if postJsonAnnounce:
postJsonObject = postJsonAnnounce postJsonObject = postJsonAnnounce
if postJsonObject['type'] != 'Create': if postJsonObject['type'] != 'Create':
@ -3500,7 +3503,8 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,
def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str, def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
raw: bool, simple: bool, proxyType: str, raw: bool, simple: bool, proxyType: str,
port: int, httpPrefix: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str) -> None: debug: bool, projectVersion: str,
systemLanguage: str) -> None:
""" This is really just for test purposes """ This is really just for test purposes
""" """
print('Starting new session for getting public posts') print('Starting new session for getting public posts')
@ -3536,13 +3540,14 @@ def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
_getPosts(session, personUrl, 30, maxMentions, maxEmoji, _getPosts(session, personUrl, 30, maxMentions, maxEmoji,
maxAttachments, federationList, maxAttachments, federationList,
personCache, raw, simple, debug, personCache, raw, simple, debug,
projectVersion, httpPrefix, domain) projectVersion, httpPrefix, domain, systemLanguage)
def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str, def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str, proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str, debug: bool, projectVersion: str,
wordFrequency: {}, domainList=[]) -> []: wordFrequency: {}, domainList: [],
systemLanguage: str) -> []:
""" Returns a list of domains referenced within public posts """ Returns a list of domains referenced within public posts
""" """
if not session: if not session:
@ -3580,7 +3585,7 @@ def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str,
maxAttachments, federationList, maxAttachments, federationList,
personCache, debug, personCache, debug,
projectVersion, httpPrefix, domain, projectVersion, httpPrefix, domain,
wordFrequency, domainList) wordFrequency, domainList, systemLanguage)
postDomains.sort() postDomains.sort()
return postDomains return postDomains
@ -3622,7 +3627,7 @@ def downloadFollowCollection(followType: str,
def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str, def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str, proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str, debug: bool, projectVersion: str,
wordFrequency: {}) -> []: wordFrequency: {}, systemLanguage: str) -> []:
""" Returns a dict of domains referenced within public posts """ Returns a dict of domains referenced within public posts
""" """
if not session: if not session:
@ -3661,7 +3666,7 @@ def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str,
maxAttachments, federationList, maxAttachments, federationList,
personCache, debug, personCache, debug,
projectVersion, httpPrefix, domain, projectVersion, httpPrefix, domain,
wordFrequency, []) wordFrequency, [], systemLanguage)
postDomains.sort() postDomains.sort()
domainsInfo = {} domainsInfo = {}
for d in postDomains: for d in postDomains:
@ -3687,7 +3692,8 @@ def getPublicPostDomainsBlocked(session, baseDir: str,
nickname: str, domain: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str, proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str, debug: bool, projectVersion: str,
wordFrequency: {}, domainList=[]) -> []: wordFrequency: {}, domainList: [],
systemLanguage: str) -> []:
""" Returns a list of domains referenced within public posts which """ Returns a list of domains referenced within public posts which
are globally blocked on this instance are globally blocked on this instance
""" """
@ -3695,7 +3701,7 @@ def getPublicPostDomainsBlocked(session, baseDir: str,
getPublicPostDomains(session, baseDir, nickname, domain, getPublicPostDomains(session, baseDir, nickname, domain,
proxyType, port, httpPrefix, proxyType, port, httpPrefix,
debug, projectVersion, debug, projectVersion,
wordFrequency, domainList) wordFrequency, domainList, systemLanguage)
if not postDomains: if not postDomains:
return [] return []
@ -3743,7 +3749,8 @@ def checkDomains(session, baseDir: str,
nickname: str, domain: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str, proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str, debug: bool, projectVersion: str,
maxBlockedDomains: int, singleCheck: bool) -> None: maxBlockedDomains: int, singleCheck: bool,
systemLanguage: str) -> None:
"""Checks follower accounts for references to globally blocked domains """Checks follower accounts for references to globally blocked domains
""" """
wordFrequency = {} wordFrequency = {}
@ -3771,7 +3778,8 @@ def checkDomains(session, baseDir: str,
nonMutualDomain, nonMutualDomain,
proxyType, port, httpPrefix, proxyType, port, httpPrefix,
debug, projectVersion, debug, projectVersion,
wordFrequency, []) wordFrequency, [],
systemLanguage)
if blockedDomains: if blockedDomains:
if len(blockedDomains) > maxBlockedDomains: if len(blockedDomains) > maxBlockedDomains:
followerWarningStr += handle + '\n' followerWarningStr += handle + '\n'
@ -3791,7 +3799,8 @@ def checkDomains(session, baseDir: str,
nonMutualDomain, nonMutualDomain,
proxyType, port, httpPrefix, proxyType, port, httpPrefix,
debug, projectVersion, debug, projectVersion,
wordFrequency, []) wordFrequency, [],
systemLanguage)
if blockedDomains: if blockedDomains:
print(handle) print(handle)
for d in blockedDomains: for d in blockedDomains:
@ -3889,7 +3898,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str,
postJsonObject: {}, projectVersion: str, postJsonObject: {}, projectVersion: str,
translate: {}, YTReplacementDomain: str, translate: {}, YTReplacementDomain: str,
allowLocalNetworkAccess: bool, allowLocalNetworkAccess: bool,
recentPostsCache: {}, debug: bool) -> {}: recentPostsCache: {}, debug: bool,
systemLanguage: str) -> {}:
"""Download the post referenced by an announce """Download the post referenced by an announce
""" """
if not postJsonObject.get('object'): if not postJsonObject.get('object'):
@ -4074,7 +4084,7 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str,
recentPostsCache) recentPostsCache)
return None return None
postJsonObject = announcedJson postJsonObject = announcedJson
replaceYouTube(postJsonObject, YTReplacementDomain) replaceYouTube(postJsonObject, YTReplacementDomain, systemLanguage)
if saveJson(postJsonObject, announceFilename): if saveJson(postJsonObject, announceFilename):
return postJsonObject return postJsonObject
return None return None

2
schedule.py
View File

@ -112,7 +112,7 @@ def _updatePostSchedule(baseDir: str, handle: str, httpd,
httpd.YTReplacementDomain, httpd.YTReplacementDomain,
httpd.showPublishedDateOnly, httpd.showPublishedDateOnly,
httpd.allowLocalNetworkAccess, httpd.allowLocalNetworkAccess,
httpd.city): httpd.city, httpd.systemLanguage):
indexLines.remove(line) indexLines.remove(line)
os.remove(postFilename) os.remove(postFilename)
continue continue

5
socnet.py
View File

@ -17,7 +17,8 @@ from utils import getFullDomain
def instancesGraph(baseDir: str, handles: str, def instancesGraph(baseDir: str, handles: str,
proxyType: str, proxyType: str,
port: int, httpPrefix: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str) -> str: debug: bool, projectVersion: str,
systemLanguage: str) -> str:
""" Returns a dot graph of federating instances """ Returns a dot graph of federating instances
based upon a few sample handles. based upon a few sample handles.
The handles argument should contain a comma separated list The handles argument should contain a comma separated list
@ -74,7 +75,7 @@ def instancesGraph(baseDir: str, handles: str,
maxAttachments, federationList, maxAttachments, federationList,
personCache, debug, personCache, debug,
projectVersion, httpPrefix, domain, projectVersion, httpPrefix, domain,
wordFrequency, []) wordFrequency, [], systemLanguage)
postDomains.sort() postDomains.sort()
for fedDomain in postDomains: for fedDomain in postDomains:
dotLineStr = ' "' + domain + '" -> "' + fedDomain + '";\n' dotLineStr = ' "' + domain + '" -> "' + fedDomain + '";\n'

6
tests.py
View File

@ -926,6 +926,8 @@ def testPostMessageBetweenServers():
assert receivedJson assert receivedJson
assert 'Why is a mouse when it spins?' in \ assert 'Why is a mouse when it spins?' in \
receivedJson['object']['content'] receivedJson['object']['content']
assert 'Why is a mouse when it spins?' in \
receivedJson['object']['contentMap'][systemLanguage]
assert 'यह एक परीक्षण है' in receivedJson['object']['content'] assert 'यह एक परीक्षण है' in receivedJson['object']['content']
print('Check that message received from Alice contains an attachment') print('Check that message received from Alice contains an attachment')
assert receivedJson['object']['attachment'] assert receivedJson['object']['attachment']
@ -2913,6 +2915,7 @@ def _testReplyToPublicPost() -> None:
'<a href=\"https://rat.site/@ninjarodent\" ' + \ '<a href=\"https://rat.site/@ninjarodent\" ' + \
'class=\"u-url mention\">@<span>ninjarodent</span>' + \ 'class=\"u-url mention\">@<span>ninjarodent</span>' + \
'</a></span> This is a test.</p>' '</a></span> This is a test.</p>'
reply['object']['contentMap'][systemLanguage] = reply['object']['content']
assert reply['object']['tag'][0]['type'] == 'Mention' assert reply['object']['tag'][0]['type'] == 'Mention'
assert reply['object']['tag'][0]['name'] == '@ninjarodent@rat.site' assert reply['object']['tag'][0]['name'] == '@ninjarodent@rat.site'
assert reply['object']['tag'][0]['href'] == \ assert reply['object']['tag'][0]['href'] == \
@ -3444,6 +3447,8 @@ def _testLinksWithinPost() -> None:
'rel="nofollow noopener noreferrer" target="_blank">' + \ 'rel="nofollow noopener noreferrer" target="_blank">' + \
'<span class="invisible">https://</span>' + \ '<span class="invisible">https://</span>' + \
'<span class="ellipsis">freedombone.net</span></a></p>' '<span class="ellipsis">freedombone.net</span></a></p>'
assert postJsonObject['object']['content'] == \
postJsonObject['object']['contentMap'][systemLanguage]
content = "<p>Some text</p><p>Other text</p><p>More text</p>" + \ content = "<p>Some text</p><p>Other text</p><p>More text</p>" + \
"<pre><code>Errno::EOHNOES (No such file or rodent @ " + \ "<pre><code>Errno::EOHNOES (No such file or rodent @ " + \
@ -3467,6 +3472,7 @@ def _testLinksWithinPost() -> None:
testEventDate, testEventTime, testLocation, testEventDate, testEventTime, testLocation,
testIsArticle, systemLanguage) testIsArticle, systemLanguage)
assert postJsonObject['object']['content'] == content assert postJsonObject['object']['content'] == content
assert postJsonObject['object']['contentMap'][systemLanguage] == content
def _testMastoApi(): def _testMastoApi():

37
utils.py
View File

@ -42,6 +42,8 @@ def getContentFromPost(postJsonObject: {}, systemLanguage: str) -> str:
if thisPostJson['contentMap'].get(systemLanguage): if thisPostJson['contentMap'].get(systemLanguage):
if isinstance(thisPostJson['contentMap'][systemLanguage], str): if isinstance(thisPostJson['contentMap'][systemLanguage], str):
return thisPostJson['contentMap'][systemLanguage] return thisPostJson['contentMap'][systemLanguage]
if not isinstance(content, str):
return ''
return content return content
@ -1732,41 +1734,6 @@ def getCSS(baseDir: str, cssFilename: str, cssCache: {}) -> str:
return None return None
def isEventPost(messageJson: {}) -> bool:
"""Is the given post a mobilizon-type event activity?
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
"""
if not messageJson.get('id'):
return False
if not messageJson.get('actor'):
return False
if not hasObjectDict(messageJson):
return False
if not messageJson['object'].get('type'):
return False
if messageJson['object']['type'] != 'Event':
return False
print('Event arriving')
if not messageJson['object'].get('startTime'):
print('No event start time')
return False
if not messageJson['object'].get('actor'):
print('No event actor')
return False
if not messageJson['object'].get('content'):
print('No event content')
return False
if not messageJson['object'].get('name'):
print('No event name')
return False
if not messageJson['object'].get('uuid'):
print('No event UUID')
return False
print('Event detected')
return True
def isBlogPost(postJsonObject: {}) -> bool: def isBlogPost(postJsonObject: {}) -> bool:
"""Is the given post a blog post? """Is the given post a blog post?
""" """

6
webapp_column_right.py
View File

@ -11,6 +11,7 @@ import os
from datetime import datetime from datetime import datetime
from content import removeLongWords from content import removeLongWords
from content import limitRepeatedWords from content import limitRepeatedWords
from utils import getContentFromPost
from utils import removeHtml from utils import removeHtml
from utils import locatePost from utils import locatePost
from utils import loadJson from utils import loadJson
@ -636,7 +637,8 @@ def htmlEditNewswire(cssCache: {}, translate: {}, baseDir: str, path: str,
def htmlEditNewsPost(cssCache: {}, translate: {}, baseDir: str, path: str, def htmlEditNewsPost(cssCache: {}, translate: {}, baseDir: str, path: str,
domain: str, port: int, domain: str, port: int,
httpPrefix: str, postUrl: str) -> str: httpPrefix: str, postUrl: str,
systemLanguage: str) -> str:
"""Edits a news post on the news/features timeline """Edits a news post on the news/features timeline
""" """
if '/users/' not in path: if '/users/' not in path:
@ -696,7 +698,7 @@ def htmlEditNewsPost(cssCache: {}, translate: {}, baseDir: str, path: str,
' <input type="text" name="newsPostTitle" value="' + \ ' <input type="text" name="newsPostTitle" value="' + \
newsPostTitle + '"><br>\n' newsPostTitle + '"><br>\n'
newsPostContent = postJsonObject['object']['content'] newsPostContent = getContentFromPost(postJsonObject, systemLanguage)
editNewsPostForm += \ editNewsPostForm += \
' <textarea id="message" name="editedNewsPost" ' + \ ' <textarea id="message" name="editedNewsPost" ' + \
'style="height:600px" spellcheck="true">' + \ 'style="height:600px" spellcheck="true">' + \

5
webapp_moderation.py
View File

@ -69,7 +69,8 @@ def htmlModeration(cssCache: {}, defaultTimeline: str,
def htmlAccountInfo(cssCache: {}, translate: {}, def htmlAccountInfo(cssCache: {}, translate: {},
baseDir: str, httpPrefix: str, baseDir: str, httpPrefix: str,
nickname: str, domain: str, port: int, nickname: str, domain: str, port: int,
searchHandle: str, debug: bool) -> str: searchHandle: str, debug: bool,
systemLanguage: str) -> str:
"""Shows which domains a search handle interacts with. """Shows which domains a search handle interacts with.
This screen is shown if a moderator enters a handle and selects info This screen is shown if a moderator enters a handle and selects info
on the moderation screen on the moderation screen
@ -111,7 +112,7 @@ def htmlAccountInfo(cssCache: {}, translate: {},
baseDir, searchNickname, searchDomain, baseDir, searchNickname, searchDomain,
proxyType, searchPort, proxyType, searchPort,
httpPrefix, debug, httpPrefix, debug,
__version__, wordFrequency) __version__, wordFrequency, systemLanguage)
# get a list of any blocked followers # get a list of any blocked followers
followersList = \ followersList = \

14
webapp_post.py
View File

@ -45,7 +45,6 @@ from utils import updateRecentPostsCache
from utils import removeIdEnding from utils import removeIdEnding
from utils import getNicknameFromActor from utils import getNicknameFromActor
from utils import getDomainFromActor from utils import getDomainFromActor
from utils import isEventPost
from utils import acctDir from utils import acctDir
from content import limitRepeatedWords from content import limitRepeatedWords
from content import replaceEmojiFromTags from content import replaceEmojiFromTags
@ -1310,7 +1309,8 @@ def individualPostAsHtml(allowDownloads: bool,
projectVersion, translate, projectVersion, translate,
YTReplacementDomain, YTReplacementDomain,
allowLocalNetworkAccess, allowLocalNetworkAccess,
recentPostsCache, False) recentPostsCache, False,
systemLanguage)
if not postJsonAnnounce: if not postJsonAnnounce:
# if the announce could not be downloaded then mark it as rejected # if the announce could not be downloaded then mark it as rejected
rejectPostId(baseDir, nickname, domain, postJsonObject['id'], rejectPostId(baseDir, nickname, domain, postJsonObject['id'],
@ -1415,13 +1415,11 @@ def individualPostAsHtml(allowDownloads: bool,
_logPostTiming(enableTimingLog, postStartTime, '10') _logPostTiming(enableTimingLog, postStartTime, '10')
isEvent = isEventPost(postJsonObject)
_logPostTiming(enableTimingLog, postStartTime, '11')
editStr = _getEditIconHtml(baseDir, nickname, domainFull, editStr = _getEditIconHtml(baseDir, nickname, domainFull,
postJsonObject, actorNickname, postJsonObject, actorNickname,
translate, isEvent) translate, False)
_logPostTiming(enableTimingLog, postStartTime, '11')
announceStr = \ announceStr = \
_getAnnounceIconHtml(isAnnounced, _getAnnounceIconHtml(isAnnounced,
@ -1588,6 +1586,8 @@ def individualPostAsHtml(allowDownloads: bool,
if postJsonObject['object'].get('cipherText'): if postJsonObject['object'].get('cipherText'):
postJsonObject['object']['content'] = \ postJsonObject['object']['content'] = \
E2EEdecryptMessageFromDevice(postJsonObject['object']) E2EEdecryptMessageFromDevice(postJsonObject['object'])
postJsonObject['object']['contentMap'][systemLanguage] = \
postJsonObject['object']['content']
contentStr = getContentFromPost(postJsonObject, systemLanguage) contentStr = getContentFromPost(postJsonObject, systemLanguage)
if not contentStr: if not contentStr:

6
webapp_search.py
View File

@ -11,6 +11,7 @@ import os
from shutil import copyfile from shutil import copyfile
import urllib.parse import urllib.parse
from datetime import datetime from datetime import datetime
from utils import getContentFromPost
from utils import isAccountDir from utils import isAccountDir
from utils import getConfigParam from utils import getConfigParam
from utils import getFullDomain from utils import getFullDomain
@ -821,7 +822,7 @@ def rssHashtagSearch(nickname: str, domain: str, port: int,
postsPerPage: int, postsPerPage: int,
session, cachedWebfingers: {}, personCache: {}, session, cachedWebfingers: {}, personCache: {},
httpPrefix: str, projectVersion: str, httpPrefix: str, projectVersion: str,
YTReplacementDomain: str) -> str: YTReplacementDomain: str, systemLanguage: str) -> str:
"""Show an rss feed for a hashtag """Show an rss feed for a hashtag
""" """
if hashtag.startswith('#'): if hashtag.startswith('#'):
@ -902,7 +903,8 @@ def rssHashtagSearch(nickname: str, domain: str, port: int,
' <title>' + \ ' <title>' + \
postJsonObject['object']['summary'] + \ postJsonObject['object']['summary'] + \
'</title>' '</title>'
description = postJsonObject['object']['content'] description = \
getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(description) description = firstParagraphFromString(description)
hashtagFeed += \ hashtagFeed += \
' <description>' + description + '</description>' ' <description>' + description + '</description>'