indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Improve handling of contentMap within posts

merge-requests/20/merge
Bob Mottram 2021-07-18 15:15:16 +01:00
parent e3e1716f1f
commit 50096bff11
19 changed files with 245 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
blog.py
View File

@ -16,6 +16,7 @@ from webapp_utils import htmlHeaderWithBlogMarkup
from webapp_utils import htmlFooter
from webapp_utils import getPostAttachmentsAsHtml
from webapp_media import addEmbeddedElements
from utils import getContentFromPost
from utils import isAccountDir
from utils import removeHtml
from utils import getConfigParam
@ -164,7 +165,8 @@ def _htmlBlogPostContent(authorized: bool,
postJsonObject: {},
handle: str, restrictToDomain: bool,
peertubeInstances: [],
blogSeparator='<hr>') -> str:
systemLanguage: str,
blogSeparator: str = '<hr>') -> str:
"""Returns the content for a single blog post
"""
linkedAuthor = False
@ -235,9 +237,9 @@ def _htmlBlogPostContent(authorized: bool,
if attachmentStr:
blogStr += '<br><center>' + attachmentStr + '</center>'
if postJsonObject['object'].get('content'):
contentStr = addEmbeddedElements(translate,
postJsonObject['object']['content'],
jsonContent = getContentFromPost(postJsonObject, systemLanguage)
if jsonContent:
contentStr = addEmbeddedElements(translate, jsonContent,
peertubeInstances)
if postJsonObject['object'].get('tag'):
contentStr = replaceEmojiFromTags(contentStr,
@ -312,7 +314,8 @@ def _htmlBlogPostRSS2(authorized: bool,
baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, domainFull: str,
postJsonObject: {},
handle: str, restrictToDomain: bool) -> str:
handle: str, restrictToDomain: bool,
systemLanguage: str) -> str:
"""Returns the RSS version 2 feed for a single blog post
"""
rssStr = ''
@ -327,7 +330,7 @@ def _htmlBlogPostRSS2(authorized: bool,
pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
titleStr = postJsonObject['object']['summary']
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content)
rssStr = ' <item>'
rssStr += ' <title>' + titleStr + '</title>'
@ -343,7 +346,8 @@ def _htmlBlogPostRSS3(authorized: bool,
baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, domainFull: str,
postJsonObject: {},
handle: str, restrictToDomain: bool) -> str:
handle: str, restrictToDomain: bool,
systemLanguage: str) -> str:
"""Returns the RSS version 3 feed for a single blog post
"""
rssStr = ''
@ -358,7 +362,7 @@ def _htmlBlogPostRSS3(authorized: bool,
pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
titleStr = postJsonObject['object']['summary']
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content)
rssStr = 'title: ' + titleStr + '\n'
rssStr += 'link: ' + messageLink + '\n'
@ -379,10 +383,10 @@ def _htmlBlogRemoveCwButton(blogStr: str, translate: {}) -> str:
return blogStr
def _getSnippetFromBlogContent(postJsonObject: {}) -> str:
def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str:
"""Returns a snippet of text from the blog post as a preview
"""
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
if '<p>' in content:
content = content.split('<p>', 1)[1]
if '</p>' in content:
@ -412,7 +416,7 @@ def htmlBlogPost(authorized: bool,
getConfigParam(baseDir, 'instanceTitle')
published = postJsonObject['object']['published']
title = postJsonObject['object']['summary']
snippet = _getSnippetFromBlogContent(postJsonObject)
snippet = _getSnippetFromBlogContent(postJsonObject, systemLanguage)
blogStr = htmlHeaderWithBlogMarkup(cssFilename, instanceTitle,
httpPrefix, domainFull, nickname,
systemLanguage, published,
@ -424,7 +428,7 @@ def htmlBlogPost(authorized: bool,
nickname, domain,
domainFull, postJsonObject,
None, False,
peertubeInstances)
peertubeInstances, systemLanguage)
# show rss links
blogStr += '<p class="rssfeed">'
@ -452,7 +456,7 @@ def htmlBlogPage(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int,
peertubeInstances: []) -> str:
peertubeInstances: [], systemLanguage: str) -> str:
"""Returns a html blog page containing posts
"""
if ' ' in nickname or '@' in nickname or \
@ -514,7 +518,8 @@ def htmlBlogPage(authorized: bool, session,
nickname, domain,
domainFull, item,
None, True,
peertubeInstances)
peertubeInstances,
systemLanguage)
if len(timelineJson['orderedItems']) >= noOfItems:
blogStr += navigateStr
@ -542,7 +547,7 @@ def htmlBlogPageRSS2(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int,
includeHeader: bool) -> str:
includeHeader: bool, systemLanguage: str) -> str:
"""Returns an RSS version 2 feed containing posts
"""
if ' ' in nickname or '@' in nickname or \
@ -585,7 +590,7 @@ def htmlBlogPageRSS2(authorized: bool, session,
httpPrefix, translate,
nickname, domain,
domainFull, item,
None, True)
None, True, systemLanguage)
if includeHeader:
return blogRSS2 + rss2Footer()
@ -596,7 +601,8 @@ def htmlBlogPageRSS2(authorized: bool, session,
def htmlBlogPageRSS3(authorized: bool, session,
baseDir: str, httpPrefix: str, translate: {},
nickname: str, domain: str, port: int,
noOfItems: int, pageNumber: int) -> str:
noOfItems: int, pageNumber: int,
systemLanguage: str) -> str:
"""Returns an RSS version 3 feed containing posts
"""
if ' ' in nickname or '@' in nickname or \
@ -630,7 +636,8 @@ def htmlBlogPageRSS3(authorized: bool, session,
httpPrefix, translate,
nickname, domain,
domainFull, item,
None, True)
None, True,
systemLanguage)
return blogRSS3
@ -670,7 +677,7 @@ def htmlBlogView(authorized: bool,
session, baseDir: str, httpPrefix: str,
translate: {}, domain: str, port: int,
noOfItems: int,
peertubeInstances: []) -> str:
peertubeInstances: [], systemLanguage: str) -> str:
"""Show the blog main page
"""
blogStr = ''
@ -688,7 +695,8 @@ def htmlBlogView(authorized: bool,
return htmlBlogPage(authorized, session,
baseDir, httpPrefix, translate,
nickname, domain, port,
noOfItems, 1, peertubeInstances)
noOfItems, 1, peertubeInstances,
systemLanguage)
domainFull = getFullDomain(domain, port)
@ -714,7 +722,7 @@ def htmlEditBlog(mediaInstance: bool, translate: {},
path: str,
pageNumber: int,
nickname: str, domain: str,
postUrl: str) -> str:
postUrl: str, systemLanguage: str) -> str:
"""Edit a blog post after it was created
"""
postFilename = locatePost(baseDir, nickname, domain, postUrl)
@ -832,7 +840,7 @@ def htmlEditBlog(mediaInstance: bool, translate: {},
placeholderMessage + '</label>'
messageBoxHeight = 800
contentStr = postJsonObject['object']['content']
contentStr = getContentFromPost(postJsonObject, systemLanguage)
contentStr = contentStr.replace('<p>', '').replace('</p>', '\n')
editBlogForm += \

60
daemon.py
View File

@ -208,6 +208,7 @@ from shares import addShare
from shares import removeShare
from shares import expireShares
from categories import setHashtagCategory
from utils import getContentFromPost
from utils import acctDir
from utils import getImageExtensionFromMimeType
from utils import getImageMimeType
@ -1110,7 +1111,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain,
self.server.showPublishedDateOnly,
self.server.allowLocalNetworkAccess,
city)
city, self.server.systemLanguage)
def _postToOutboxThread(self, messageJson: {}) -> bool:
"""Creates a thread to send a post
@ -1308,7 +1309,8 @@ class PubServer(BaseHTTPRequestHandler):
headersDict,
self.path,
self.server.debug,
self.server.blockedCache)
self.server.blockedCache,
self.server.systemLanguage)
if queueFilename:
# add json to the queue
if queueFilename not in self.server.inboxQueue:
@ -1696,7 +1698,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain,
self.server.port,
searchHandle,
self.server.debug)
self.server.debug,
self.server.systemLanguage)
else:
msg = \
htmlModerationInfo(self.server.cssCache,
@ -2344,7 +2347,8 @@ class PubServer(BaseHTTPRequestHandler):
domain,
self.server.port,
optionsActor,
self.server.debug).encode('utf-8')
self.server.debug,
self.server.systemLanguage).encode('utf-8')
msglen = len(msg)
self._set_headers('text/html', msglen,
cookie, callingDomain)
@ -3910,6 +3914,8 @@ class PubServer(BaseHTTPRequestHandler):
newsPostTitle
postJsonObject['object']['content'] = \
newsPostContent
contentMap = postJsonObject['object']['contentMap']
contentMap[self.server.systemLanguage] = newsPostContent
# update newswire
pubDate = postJsonObject['object']['published']
publishedDate = \
@ -5619,7 +5625,8 @@ class PubServer(BaseHTTPRequestHandler):
domain,
port,
maxPostsInRSSFeed, 1,
True)
True,
self.server.systemLanguage)
if msg is not None:
msg = msg.encode('utf-8')
msglen = len(msg)
@ -5674,7 +5681,8 @@ class PubServer(BaseHTTPRequestHandler):
domain,
port,
maxPostsInRSSFeed, 1,
False)
False,
self.server.systemLanguage)
break
if msg:
msg = rss2Header(httpPrefix,
@ -5776,7 +5784,7 @@ class PubServer(BaseHTTPRequestHandler):
baseDir: str, httpPrefix: str,
domain: str, port: int, proxyType: str,
GETstartTime, GETtimings: {},
debug: bool) -> None:
debug: bool, systemLanguage: str) -> None:
"""Returns an RSS3 feed
"""
nickname = path.split('/blog/')[1]
@ -5800,7 +5808,8 @@ class PubServer(BaseHTTPRequestHandler):
baseDir, httpPrefix,
self.server.translate,
nickname, domain, port,
maxPostsInRSSFeed, 1)
maxPostsInRSSFeed, 1,
systemLanguage)
if msg is not None:
msg = msg.encode('utf-8')
msglen = len(msg)
@ -6233,7 +6242,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.personCache,
httpPrefix,
self.server.projectVersion,
self.server.YTReplacementDomain)
self.server.YTReplacementDomain,
self.server.systemLanguage)
if hashtagStr:
msg = hashtagStr.encode('utf-8')
msglen = len(msg)
@ -9767,7 +9777,8 @@ class PubServer(BaseHTTPRequestHandler):
nickname,
domain, port,
maxPostsInBlogsFeed, pageNumber,
self.server.peertubeInstances)
self.server.peertubeInstances,
self.server.systemLanguage)
if msg is not None:
msg = msg.encode('utf-8')
msglen = len(msg)
@ -10384,7 +10395,8 @@ class PubServer(BaseHTTPRequestHandler):
translate, baseDir,
path, domain, port,
httpPrefix,
postUrl).encode('utf-8')
postUrl,
self.server.systemLanguage).encode('utf-8')
if msg:
msglen = len(msg)
self._set_headers('text/html', msglen,
@ -10732,7 +10744,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.port,
self.server.proxyType,
GETstartTime, GETtimings,
self.server.debug)
self.server.debug,
self.server.systemLanguage)
return
usersInPath = False
@ -10894,7 +10907,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain,
self.server.port,
maxPostsInBlogsFeed,
self.server.peertubeInstances)
self.server.peertubeInstances,
self.server.systemLanguage)
if msg is not None:
msg = msg.encode('utf-8')
msglen = len(msg)
@ -12265,7 +12279,7 @@ class PubServer(BaseHTTPRequestHandler):
self.path,
replyPageNumber,
nickname, self.server.domain,
postUrl)
postUrl, self.server.systemLanguage)
if msg:
msg = msg.encode('utf-8')
msglen = len(msg)
@ -12643,7 +12657,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain,
self.server.port,
searchHandle,
self.server.debug)
self.server.debug,
self.server.systemLanguage)
msg = msg.encode('utf-8')
msglen = len(msg)
self._login_headers('text/html',
@ -12677,7 +12692,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.domain,
self.server.port,
searchHandle,
self.server.debug)
self.server.debug,
self.server.systemLanguage)
msg = msg.encode('utf-8')
msglen = len(msg)
self._login_headers('text/html',
@ -13122,9 +13138,11 @@ class PubServer(BaseHTTPRequestHandler):
if fields['schedulePost']:
return 1
if pinToProfile:
contentStr = \
getContentFromPost(messageJson,
self.server.systemLanguage)
pinPost(self.server.baseDir,
nickname, self.server.domain,
messageJson['object']['content'])
nickname, self.server.domain, contentStr)
return 1
if self._postToOutbox(messageJson, __version__, nickname):
populateReplies(self.server.baseDir,
@ -13251,6 +13269,9 @@ class PubServer(BaseHTTPRequestHandler):
tags, 'content')
postJsonObject['object']['content'] = fields['message']
contentMap = postJsonObject['object']['contentMap']
contentMap[self.server.systemLanguage] = \
fields['message']
imgDescription = ''
if fields.get('imageDescription'):
@ -13274,7 +13295,8 @@ class PubServer(BaseHTTPRequestHandler):
city)
replaceYouTube(postJsonObject,
self.server.YTReplacementDomain)
self.server.YTReplacementDomain,
self.server.systemLanguage)
saveJson(postJsonObject, postFilename)
# also save to the news actor
if nickname != 'news':

25
desktop_client.py
View File

@ -16,6 +16,7 @@ import webbrowser
import urllib.parse
from pathlib import Path
from random import randint
from utils import getContentFromPost
from utils import hasObjectDict
from utils import getFullDomain
from utils import isDM
@ -689,15 +690,16 @@ def _readLocalBoxPost(session, nickname: str, domain: str,
__version__, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, False)
recentPostsCache, False,
systemLanguage)
if postJsonObject2:
if hasObjectDict(postJsonObject2):
if postJsonObject2['object'].get('attributedTo') and \
postJsonObject2['object'].get('content'):
attributedTo = postJsonObject2['object']['attributedTo']
content = postJsonObject2['object']['content']
if isinstance(attributedTo, str) and \
isinstance(content, str):
content = \
getContentFromPost(postJsonObject2, systemLanguage)
if isinstance(attributedTo, str) and content:
actor = attributedTo
nameStr += ' ' + translate['announces'] + ' ' + \
getNicknameFromActor(actor)
@ -721,7 +723,7 @@ def _readLocalBoxPost(session, nickname: str, domain: str,
attributedTo = postJsonObject['object']['attributedTo']
if not attributedTo:
return {}
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
if not isinstance(attributedTo, str) or \
not isinstance(content, str):
return {}
@ -1044,7 +1046,8 @@ def _desktopShowBox(indent: str,
published = _formatPublished(postJsonObject['published'])
content = _textOnlyContent(postJsonObject['object']['content'])
contentStr = getContentFromPost(postJsonObject, systemLanguage)
content = _textOnlyContent(contentStr)
if boxName != 'dm':
if isDM(postJsonObject):
content = '📧' + content
@ -2321,11 +2324,13 @@ def runDesktopClient(baseDir: str, proxyType: str, httpPrefix: str,
__version__, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, False)
recentPostsCache, False,
systemLanguage)
if postJsonObject2:
postJsonObject = postJsonObject2
if postJsonObject:
content = postJsonObject['object']['content']
content = \
getContentFromPost(postJsonObject, systemLanguage)
messageStr, detectedLinks = \
speakableText(baseDir, content, translate)
linkOpened = False
@ -2381,7 +2386,9 @@ def runDesktopClient(baseDir: str, proxyType: str, httpPrefix: str,
print('')
if postJsonObject['object'].get('summary'):
print(postJsonObject['object']['summary'])
print(postJsonObject['object']['content'])
contentStr = getContentFromPost(postJsonObject,
systemLanguage)
print(contentStr)
print('')
sayStr = 'Confirm delete, yes or no?'
_sayCommand(sayStr, sayStr, screenreader,

26
epicyon.py
View File

@ -631,9 +631,11 @@ if args.posts:
args.port = 80
elif args.gnunet:
proxyType = 'gnunet'
if not args.language:
args.language = 'en'
getPublicPostsOfPerson(baseDir, nickname, domain, False, True,
proxyType, args.port, httpPrefix, debug,
__version__)
__version__, args.language)
sys.exit()
if args.postDomains:
@ -663,12 +665,15 @@ if args.postDomains:
proxyType = 'gnunet'
wordFrequency = {}
domainList = []
if not args.language:
args.language = 'en'
domainList = getPublicPostDomains(None,
baseDir, nickname, domain,
proxyType, args.port,
httpPrefix, debug,
__version__,
wordFrequency, domainList)
wordFrequency, domainList,
args.language)
for postDomain in domainList:
print(postDomain)
sys.exit()
@ -703,12 +708,15 @@ if args.postDomainsBlocked:
proxyType = 'gnunet'
wordFrequency = {}
domainList = []
if not args.language:
args.language = 'en'
domainList = getPublicPostDomainsBlocked(None,
baseDir, nickname, domain,
proxyType, args.port,
httpPrefix, debug,
__version__,
wordFrequency, domainList)
wordFrequency, domainList,
args.language)
for postDomain in domainList:
print(postDomain)
sys.exit()
@ -741,12 +749,14 @@ if args.checkDomains:
elif args.gnunet:
proxyType = 'gnunet'
maxBlockedDomains = 0
if not args.language:
args.language = 'en'
checkDomains(None,
baseDir, nickname, domain,
proxyType, args.port,
httpPrefix, debug,
__version__,
maxBlockedDomains, False)
maxBlockedDomains, False, args.language)
sys.exit()
if args.socnet:
@ -758,10 +768,12 @@ if args.socnet:
if not args.http:
args.port = 443
proxyType = 'tor'
if not args.language:
args.language = 'en'
dotGraph = instancesGraph(baseDir, args.socnet,
proxyType, args.port,
httpPrefix, debug,
__version__)
__version__, args.language)
try:
with open('socnet.dot', 'w+') as fp:
fp.write(dotGraph)
@ -785,9 +797,11 @@ if args.postsraw:
proxyType = 'i2p'
elif args.gnunet:
proxyType = 'gnunet'
if not args.language:
args.language = 'en'
getPublicPostsOfPerson(baseDir, nickname, domain, False, False,
proxyType, args.port, httpPrefix, debug,
__version__)
__version__, args.language)
sys.exit()
if args.json:

3
happening.py
View File

@ -55,6 +55,9 @@ def saveEventPost(baseDir: str, handle: str, postId: str,
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
"""
if not os.path.isdir(baseDir + '/accounts/' + handle):
print('WARN: Account does not exist at ' +
baseDir + '/accounts/' + handle)
calendarPath = baseDir + '/accounts/' + handle + '/calendar'
if not os.path.isdir(calendarPath):
os.mkdir(calendarPath)

92
inbox.py
View File

@ -13,6 +13,7 @@ import datetime
import time
import random
from linked_data_sig import verifyJsonSignature
from utils import getContentFromPost
from utils import acctDir
from utils import removeDomainPort
from utils import getPortFromDomain
@ -23,7 +24,6 @@ from utils import getConfigParam
from utils import hasUsersPath
from utils import validPostDate
from utils import getFullDomain
from utils import isEventPost
from utils import removeIdEnding
from utils import getProtocolPrefixes
from utils import isBlogPost
@ -351,7 +351,7 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str,
messageBytes: str,
httpHeaders: {},
postPath: str, debug: bool,
blockedCache: []) -> str:
blockedCache: [], systemLanguage: str) -> str:
"""Saves the give json to the inbox queue for the person
keyId specifies the actor sending the post
"""
@ -415,9 +415,9 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str,
replyNickname + '@' + replyDomain)
return None
if postJsonObject['object'].get('content'):
if isinstance(postJsonObject['object']['content'], str):
if isFiltered(baseDir, nickname, domain,
postJsonObject['object']['content']):
contentStr = getContentFromPost(postJsonObject, systemLanguage)
if contentStr:
if isFiltered(baseDir, nickname, domain, contentStr):
if debug:
print('WARN: post was filtered out due to content')
return None
@ -730,25 +730,6 @@ def _receiveUndo(session, baseDir: str, httpPrefix: str,
return False
def _receiveEventPost(recentPostsCache: {}, session, baseDir: str,
httpPrefix: str, domain: str, port: int,
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, messageJson: {}, federationList: [],
nickname: str, debug: bool) -> bool:
"""Receive a mobilizon-type event activity
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
"""
if not isEventPost(messageJson):
return
print('Receiving event: ' + str(messageJson['object']))
handle = getFullDomain(nickname + '@' + domain, port)
postId = removeIdEnding(messageJson['id']).replace('/', '#')
saveEventPost(baseDir, handle, postId, messageJson['object'])
def _personReceiveUpdate(baseDir: str,
domain: str, port: int,
updateNickname: str, updateDomain: str,
@ -1305,7 +1286,7 @@ def _receiveAnnounce(recentPostsCache: {},
debug: bool, translate: {},
YTReplacementDomain: str,
allowLocalNetworkAccess: bool,
themeName: str) -> bool:
themeName: str, systemLanguage: str) -> bool:
"""Receives an announce activity within the POST section of HTTPServer
"""
if messageJson['type'] != 'Announce':
@ -1392,7 +1373,8 @@ def _receiveAnnounce(recentPostsCache: {},
__version__, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, debug)
recentPostsCache, debug,
systemLanguage)
if not postJsonObject:
notInOnion = True
if onionDomain:
@ -1616,7 +1598,8 @@ def _estimateNumberOfEmoji(content: str) -> int:
def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson: {}, maxMentions: int, maxEmoji: int,
allowLocalNetworkAccess: bool, debug: bool) -> bool:
allowLocalNetworkAccess: bool, debug: bool,
systemLanguage: str) -> bool:
"""Is the content of a received post valid?
Check for bad html
Check for hellthreads
@ -1651,27 +1634,27 @@ def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson['object']['content']):
return True
if dangerousMarkup(messageJson['object']['content'],
allowLocalNetworkAccess):
contentStr = getContentFromPost(messageJson, systemLanguage)
if dangerousMarkup(contentStr, allowLocalNetworkAccess):
if messageJson['object'].get('id'):
print('REJECT ARBITRARY HTML: ' + messageJson['object']['id'])
print('REJECT ARBITRARY HTML: bad string in post - ' +
messageJson['object']['content'])
contentStr)
return False
# check (rough) number of mentions
mentionsEst = _estimateNumberOfMentions(messageJson['object']['content'])
mentionsEst = _estimateNumberOfMentions(contentStr)
if mentionsEst > maxMentions:
if messageJson['object'].get('id'):
print('REJECT HELLTHREAD: ' + messageJson['object']['id'])
print('REJECT HELLTHREAD: Too many mentions in post - ' +
messageJson['object']['content'])
contentStr)
return False
if _estimateNumberOfEmoji(messageJson['object']['content']) > maxEmoji:
if _estimateNumberOfEmoji(contentStr) > maxEmoji:
if messageJson['object'].get('id'):
print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id'])
print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' +
messageJson['object']['content'])
contentStr)
return False
# check number of tags
if messageJson['object'].get('tag'):
@ -1685,8 +1668,7 @@ def _validPostContent(baseDir: str, nickname: str, domain: str,
messageJson['object']['tag'])
return False
# check for filtered content
if isFiltered(baseDir, nickname, domain,
messageJson['object']['content']):
if isFiltered(baseDir, nickname, domain, contentStr):
print('REJECT: content filtered')
return False
if messageJson['object'].get('inReplyTo'):
@ -1926,7 +1908,8 @@ def _sendToGroupMembers(session, baseDir: str, handle: str, port: int,
postJsonObject: {},
httpPrefix: str, federationList: [],
sendThreads: [], postLog: [], cachedWebfingers: {},
personCache: {}, debug: bool) -> None:
personCache: {}, debug: bool,
systemLanguage: str) -> None:
"""When a post arrives for a group send it out to the group members
"""
followersFile = baseDir + '/accounts/' + handle + '/followers.txt'
@ -1947,9 +1930,12 @@ def _sendToGroupMembers(session, baseDir: str, handle: str, port: int,
sendingActorDomainFull = \
getFullDomain(sendingActorDomain, sendingActorPort)
senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull
if not postJsonObject['object']['content'].startswith(senderStr):
contentStr = getContentFromPost(postJsonObject, systemLanguage)
if not contentStr.startswith(senderStr):
postJsonObject['object']['content'] = \
senderStr + ' ' + postJsonObject['object']['content']
senderStr + ' ' + contentStr
postJsonObject['object']['contentMap'][systemLanguage] = \
senderStr + ' ' + contentStr
# add mention to tag list
if not postJsonObject['object']['tag']:
postJsonObject['object']['tag'] = []
@ -2367,7 +2353,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
debug, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
themeName):
themeName, systemLanguage):
if debug:
print('DEBUG: Announce accepted from ' + actor)
@ -2416,7 +2402,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
jsonObj = None
if _validPostContent(baseDir, nickname, domain,
postJsonObject, maxMentions, maxEmoji,
allowLocalNetworkAccess, debug):
allowLocalNetworkAccess, debug,
systemLanguage):
if postJsonObject.get('object'):
jsonObj = postJsonObject['object']
@ -2449,7 +2436,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
return False
# replace YouTube links, so they get less tracking data
replaceYouTube(postJsonObject, YTReplacementDomain)
replaceYouTube(postJsonObject, YTReplacementDomain, systemLanguage)
# list of indexes to be updated
updateIndexList = ['inbox']
@ -2519,7 +2506,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
nickname, domain, postJsonObject,
translate, YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, debug):
recentPostsCache, debug, systemLanguage):
# media index will be updated
updateIndexList.append('tlmedia')
if isBlogPost(postJsonObject):
@ -2613,7 +2600,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
postJsonObject,
httpPrefix, federationList, sendThreads,
postLog, cachedWebfingers, personCache,
debug)
debug, systemLanguage)
# if the post wasn't saved
if not os.path.isfile(destinationFilename):
@ -3149,23 +3136,6 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
queue.pop(0)
continue
if _receiveEventPost(recentPostsCache, session,
baseDir, httpPrefix,
domain, port,
sendThreads, postLog,
cachedWebfingers,
personCache,
queueJson['post'],
federationList,
queueJson['postNickname'],
debug):
print('Queue: Event activity accepted from ' + keyId)
if os.path.isfile(queueFilename):
os.remove(queueFilename)
if len(queue) > 0:
queue.pop(0)
continue
if _receiveUpdate(recentPostsCache, session,
baseDir, httpPrefix,
domain, port,

13
media.py
View File

@ -13,6 +13,7 @@ import subprocess
from random import randint
from hashlib import sha1
from auth import createPassword
from utils import getContentFromPost
from utils import getFullDomain
from utils import getImageExtensions
from utils import getVideoExtensions
@ -26,7 +27,8 @@ from shutil import move
from city import spoofGeolocation
def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None:
def replaceYouTube(postJsonObject: {}, replacementDomain: str,
systemLanguage: str) -> None:
"""Replace YouTube with a replacement domain
This denies Google some, but not all, tracking data
"""
@ -36,11 +38,12 @@ def replaceYouTube(postJsonObject: {}, replacementDomain: str) -> None:
return
if not postJsonObject['object'].get('content'):
return
if 'www.youtube.com' not in postJsonObject['object']['content']:
contentStr = getContentFromPost(postJsonObject, systemLanguage)
if 'www.youtube.com' not in contentStr:
return
postJsonObject['object']['content'] = \
postJsonObject['object']['content'].replace('www.youtube.com',
replacementDomain)
contentStr = contentStr.replace('www.youtube.com', replacementDomain)
postJsonObject['object']['content'] = contentStr
postJsonObject['object']['contentMap'][systemLanguage] = contentStr
def _removeMetaData(imageFilename: str, outputFilename: str) -> None:

29
newsdaemon.py
View File

@ -25,6 +25,7 @@ from newswire import getDictFromNewswire
from posts import createNewsPost
from posts import archivePostsForPerson
from content import validHashTag
from utils import getContentFromPost
from utils import removeHtml
from utils import getFullDomain
from utils import loadJson
@ -279,7 +280,7 @@ def hashtagRuleTree(operators: [],
def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
postJsonObject: {},
actionStr: str, hashtags: []) -> None:
actionStr: str, hashtags: [], systemLanguage: str) -> None:
"""Adds a hashtag via a hashtag rule
"""
addHashtag = actionStr.split('add ', 1)[1].strip()
@ -313,7 +314,7 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
hashtagHtml = \
" <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
if hashtagHtml in content:
return
@ -328,7 +329,7 @@ def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
actionStr: str, hashtags: []) -> None:
actionStr: str, hashtags: [], systemLanguage: str) -> None:
"""Removes a hashtag via a hashtag rule
"""
rmHashtag = actionStr.split('remove ', 1)[1].strip()
@ -343,10 +344,11 @@ def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
hashtagHtml = \
"<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content']
content = getContentFromPost(postJsonObject, systemLanguage)
if hashtagHtml in content:
content = content.replace(hashtagHtml, '').replace(' ', ' ')
postJsonObject['object']['content'] = content
postJsonObject['object']['contentMap'][systemLanguage] = content
rmTagObject = None
for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'):
@ -365,7 +367,8 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
cachedWebfingers: {},
federationList: [],
sendThreads: [], postLog: [],
moderated: bool, url: str) -> bool:
moderated: bool, url: str,
systemLanguage: str) -> bool:
"""Applies hashtag rules to a news post.
Returns true if the post should be saved to the news timeline
of this instance
@ -382,7 +385,7 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
# get the full text content of the post
content = ''
if postJsonObject['object'].get('content'):
content += postJsonObject['object']['content']
content += getContentFromPost(postJsonObject, systemLanguage)
if postJsonObject['object'].get('summary'):
content += ' ' + postJsonObject['object']['summary']
content = content.lower()
@ -409,11 +412,11 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
if actionStr.startswith('add '):
# add a hashtag
_hashtagAdd(baseDir, httpPrefix, domainFull,
postJsonObject, actionStr, hashtags)
postJsonObject, actionStr, hashtags, systemLanguage)
elif actionStr.startswith('remove '):
# remove a hashtag
_hashtagRemove(httpPrefix, domainFull, postJsonObject,
actionStr, hashtags)
actionStr, hashtags, systemLanguage)
elif actionStr.startswith('block') or actionStr.startswith('drop'):
# Block this item
return False
@ -627,7 +630,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
blog['object']['published'] = dateStr
blog['object']['content'] = rssDescription
blog['object']['contentMap']['en'] = rssDescription
blog['object']['contentMap'][systemLanguage] = rssDescription
domainFull = getFullDomain(domain, port)
@ -642,7 +645,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
personCache, cachedWebfingers,
federationList,
sendThreads, postLog,
moderated, url)
moderated, url, systemLanguage)
# save the post and update the index
if savePost:
@ -664,7 +667,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
"\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + \
htId + "</span></a>"
content = blog['object']['content']
content = getContentFromPost(blog, systemLanguage)
if hashtagHtml not in content:
if content.endswith('</p>'):
content = \
@ -673,6 +676,7 @@ def _convertRSStoActivityPub(baseDir: str, httpPrefix: str,
else:
content += hashtagHtml
blog['object']['content'] = content
blog['object']['contentMap'][systemLanguage] = content
# update the newswire tags if new ones have been found by
# _newswireHashtagProcessing
@ -749,7 +753,8 @@ def runNewswireDaemon(baseDir: str, httpd,
httpd.maxTags,
httpd.maxFeedItemSizeKb,
httpd.maxNewswirePosts,
httpd.maxCategoriesFeedItemSizeKb)
httpd.maxCategoriesFeedItemSizeKb,
httpd.systemLanguage)
if not httpd.newswire:
if os.path.isfile(newswireStateFilename):

16
newswire.py
View File

@ -18,6 +18,7 @@ from datetime import timezone
from collections import OrderedDict
from utils import validPostDate
from categories import setHashtagCategory
from utils import getContentFromPost
from utils import hasObjectDict
from utils import firstParagraphFromString
from utils import isPublicPost
@ -909,7 +910,7 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
newswire: {},
maxBlogsPerAccount: int,
indexFilename: str,
maxTags: int) -> None:
maxTags: int, systemLanguage: str) -> None:
"""Adds blogs for the given account to the newswire
"""
if not os.path.isfile(indexFilename):
@ -961,7 +962,8 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
votes = []
if os.path.isfile(fullPostFilename + '.votes'):
votes = loadJson(fullPostFilename + '.votes')
content = postJsonObject['object']['content']
content = \
getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(content)
description = removeHtml(description)
tagsFromPost = _getHashtagsFromPost(postJsonObject)
@ -981,7 +983,7 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
maxBlogsPerAccount: int,
maxTags: int) -> None:
maxTags: int, systemLanguage: str) -> None:
"""Adds blogs from each user account into the newswire
"""
moderationDict = {}
@ -1009,7 +1011,8 @@ def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
domain = handle.split('@')[1]
_addAccountBlogsToNewswire(baseDir, nickname, domain,
newswire, maxBlogsPerAccount,
blogsIndex, maxTags)
blogsIndex, maxTags,
systemLanguage)
break
# sort the moderation dict into chronological order, latest first
@ -1029,7 +1032,8 @@ def getDictFromNewswire(session, baseDir: str, domain: str,
maxPostsPerSource: int, maxFeedSizeKb: int,
maxTags: int, maxFeedItemSizeKb: int,
maxNewswirePosts: int,
maxCategoriesFeedItemSizeKb: int) -> {}:
maxCategoriesFeedItemSizeKb: int,
systemLanguage: str) -> {}:
"""Gets rss feeds as a dictionary from newswire file
"""
subscriptionsFilename = baseDir + '/accounts/newswire.txt'
@ -1077,7 +1081,7 @@ def getDictFromNewswire(session, baseDir: str, domain: str,
# add blogs from each user account
_addBlogsToNewswire(baseDir, domain, result,
maxPostsPerSource, maxTags)
maxPostsPerSource, maxTags, systemLanguage)
# sort into chronological order, latest first
sortedResult = OrderedDict(sorted(result.items(), reverse=True))

13
outbox.py
View File

@ -16,6 +16,7 @@ from posts import outboxMessageCreateWrap
from posts import savePostToBox
from posts import sendToFollowersThread
from posts import sendToNamedAddresses
from utils import getContentFromPost
from utils import hasObjectDict
from utils import getLocalNetworkAddresses
from utils import getFullDomain
@ -178,7 +179,7 @@ def postMessageToOutbox(session, translate: {},
YTReplacementDomain: str,
showPublishedDateOnly: bool,
allowLocalNetworkAccess: bool,
city: str) -> bool:
city: str, systemLanguage: str) -> bool:
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
@ -201,9 +202,9 @@ def postMessageToOutbox(session, translate: {},
# check that the outgoing post doesn't contain any markup
# which can be used to implement exploits
if hasObjectDict(messageJson):
if messageJson['object'].get('content'):
if dangerousMarkup(messageJson['object']['content'],
allowLocalNetworkAccess):
contentStr = getContentFromPost(messageJson, systemLanguage)
if contentStr:
if dangerousMarkup(contentStr, allowLocalNetworkAccess):
print('POST to outbox contains dangerous markup: ' +
str(messageJson))
return False
@ -264,7 +265,7 @@ def postMessageToOutbox(session, translate: {},
print('DEBUG: domain is blocked: ' + messageJson['actor'])
return False
# replace youtube, so that google gets less tracking data
replaceYouTube(messageJson, YTReplacementDomain)
replaceYouTube(messageJson, YTReplacementDomain, systemLanguage)
# https://www.w3.org/TR/activitypub/#create-activity-outbox
messageJson['object']['attributedTo'] = messageJson['actor']
if messageJson['object'].get('attachment'):
@ -390,7 +391,7 @@ def postMessageToOutbox(session, translate: {},
messageJson,
translate, YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, debug):
recentPostsCache, debug, systemLanguage):
inboxUpdateIndex('tlmedia', baseDir,
postToNickname + '@' + domain,
savedFilename, debug)

54
posts.py
View File

@ -31,6 +31,7 @@ from session import postImage
from webfinger import webfingerHandle
from httpsig import createSignedHeader
from siteactive import siteIsActive
from utils import getContentFromPost
from utils import removeDomainPort
from utils import getPortFromDomain
from utils import hasObjectDict
@ -322,7 +323,7 @@ def _getPosts(session, outboxUrl: str, maxPosts: int,
personCache: {}, raw: bool,
simple: bool, debug: bool,
projectVersion: str, httpPrefix: str,
domain: str) -> {}:
domain: str, systemLanguage: str) -> {}:
"""Gets public posts from an outbox
"""
personPosts = {}
@ -385,8 +386,8 @@ def _getPosts(session, outboxUrl: str, maxPosts: int,
if not isPublic:
continue
content = \
item['object']['content'].replace('&apos;', "'")
content = getContentFromPost(item, systemLanguage)
content = content.replace('&apos;', "'")
mentions = []
emoji = {}
@ -538,7 +539,7 @@ def getPostDomains(session, outboxUrl: str, maxPosts: int,
projectVersion: str, httpPrefix: str,
domain: str,
wordFrequency: {},
domainList=[]) -> []:
domainList: [], systemLanguage: str) -> []:
"""Returns a list of domains referenced within public posts
"""
if not outboxUrl:
@ -563,9 +564,9 @@ def getPostDomains(session, outboxUrl: str, maxPosts: int,
break
if not hasObjectDict(item):
continue
if item['object'].get('content'):
_updateWordFrequency(item['object']['content'],
wordFrequency)
contentStr = getContentFromPost(item, systemLanguage)
if contentStr:
_updateWordFrequency(contentStr, wordFrequency)
if item['object'].get('inReplyTo'):
if isinstance(item['object']['inReplyTo'], str):
postDomain, postPort = \
@ -2932,7 +2933,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str,
postJsonObject: {}, translate: {},
YTReplacementDomain: str,
allowLocalNetworkAccess: bool,
recentPostsCache: {}, debug: bool) -> bool:
recentPostsCache: {}, debug: bool,
systemLanguage: str) -> bool:
"""Returns true if the given post has attached image media
"""
if postJsonObject['type'] == 'Announce':
@ -2942,7 +2944,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str,
__version__, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, debug)
recentPostsCache, debug,
systemLanguage)
if postJsonAnnounce:
postJsonObject = postJsonAnnounce
if postJsonObject['type'] != 'Create':
@ -3500,7 +3503,8 @@ def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,
def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
raw: bool, simple: bool, proxyType: str,
port: int, httpPrefix: str,
debug: bool, projectVersion: str) -> None:
debug: bool, projectVersion: str,
systemLanguage: str) -> None:
""" This is really just for test purposes
"""
print('Starting new session for getting public posts')
@ -3536,13 +3540,14 @@ def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
_getPosts(session, personUrl, 30, maxMentions, maxEmoji,
maxAttachments, federationList,
personCache, raw, simple, debug,
projectVersion, httpPrefix, domain)
projectVersion, httpPrefix, domain, systemLanguage)
def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str,
wordFrequency: {}, domainList=[]) -> []:
wordFrequency: {}, domainList: [],
systemLanguage: str) -> []:
""" Returns a list of domains referenced within public posts
"""
if not session:
@ -3580,7 +3585,7 @@ def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str,
maxAttachments, federationList,
personCache, debug,
projectVersion, httpPrefix, domain,
wordFrequency, domainList)
wordFrequency, domainList, systemLanguage)
postDomains.sort()
return postDomains
@ -3622,7 +3627,7 @@ def downloadFollowCollection(followType: str,
def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str,
wordFrequency: {}) -> []:
wordFrequency: {}, systemLanguage: str) -> []:
""" Returns a dict of domains referenced within public posts
"""
if not session:
@ -3661,7 +3666,7 @@ def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str,
maxAttachments, federationList,
personCache, debug,
projectVersion, httpPrefix, domain,
wordFrequency, [])
wordFrequency, [], systemLanguage)
postDomains.sort()
domainsInfo = {}
for d in postDomains:
@ -3687,7 +3692,8 @@ def getPublicPostDomainsBlocked(session, baseDir: str,
nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str,
wordFrequency: {}, domainList=[]) -> []:
wordFrequency: {}, domainList: [],
systemLanguage: str) -> []:
""" Returns a list of domains referenced within public posts which
are globally blocked on this instance
"""
@ -3695,7 +3701,7 @@ def getPublicPostDomainsBlocked(session, baseDir: str,
getPublicPostDomains(session, baseDir, nickname, domain,
proxyType, port, httpPrefix,
debug, projectVersion,
wordFrequency, domainList)
wordFrequency, domainList, systemLanguage)
if not postDomains:
return []
@ -3743,7 +3749,8 @@ def checkDomains(session, baseDir: str,
nickname: str, domain: str,
proxyType: str, port: int, httpPrefix: str,
debug: bool, projectVersion: str,
maxBlockedDomains: int, singleCheck: bool) -> None:
maxBlockedDomains: int, singleCheck: bool,
systemLanguage: str) -> None:
"""Checks follower accounts for references to globally blocked domains
"""
wordFrequency = {}
@ -3771,7 +3778,8 @@ def checkDomains(session, baseDir: str,
nonMutualDomain,
proxyType, port, httpPrefix,
debug, projectVersion,
wordFrequency, [])
wordFrequency, [],
systemLanguage)
if blockedDomains:
if len(blockedDomains) > maxBlockedDomains:
followerWarningStr += handle + '\n'
@ -3791,7 +3799,8 @@ def checkDomains(session, baseDir: str,
nonMutualDomain,
proxyType, port, httpPrefix,
debug, projectVersion,
wordFrequency, [])
wordFrequency, [],
systemLanguage)
if blockedDomains:
print(handle)
for d in blockedDomains:
@ -3889,7 +3898,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str,
postJsonObject: {}, projectVersion: str,
translate: {}, YTReplacementDomain: str,
allowLocalNetworkAccess: bool,
recentPostsCache: {}, debug: bool) -> {}:
recentPostsCache: {}, debug: bool,
systemLanguage: str) -> {}:
"""Download the post referenced by an announce
"""
if not postJsonObject.get('object'):
@ -4074,7 +4084,7 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str,
recentPostsCache)
return None
postJsonObject = announcedJson
replaceYouTube(postJsonObject, YTReplacementDomain)
replaceYouTube(postJsonObject, YTReplacementDomain, systemLanguage)
if saveJson(postJsonObject, announceFilename):
return postJsonObject
return None

2
schedule.py
View File

@ -112,7 +112,7 @@ def _updatePostSchedule(baseDir: str, handle: str, httpd,
httpd.YTReplacementDomain,
httpd.showPublishedDateOnly,
httpd.allowLocalNetworkAccess,
httpd.city):
httpd.city, httpd.systemLanguage):
indexLines.remove(line)
os.remove(postFilename)
continue

5
socnet.py
View File

@ -17,7 +17,8 @@ from utils import getFullDomain
def instancesGraph(baseDir: str, handles: str,
proxyType: str,
port: int, httpPrefix: str,
debug: bool, projectVersion: str) -> str:
debug: bool, projectVersion: str,
systemLanguage: str) -> str:
""" Returns a dot graph of federating instances
based upon a few sample handles.
The handles argument should contain a comma separated list
@ -74,7 +75,7 @@ def instancesGraph(baseDir: str, handles: str,
maxAttachments, federationList,
personCache, debug,
projectVersion, httpPrefix, domain,
wordFrequency, [])
wordFrequency, [], systemLanguage)
postDomains.sort()
for fedDomain in postDomains:
dotLineStr = ' "' + domain + '" -> "' + fedDomain + '";\n'

6
tests.py
View File

@ -926,6 +926,8 @@ def testPostMessageBetweenServers():
assert receivedJson
assert 'Why is a mouse when it spins?' in \
receivedJson['object']['content']
assert 'Why is a mouse when it spins?' in \
receivedJson['object']['contentMap'][systemLanguage]
assert 'यह एक परीक्षण है' in receivedJson['object']['content']
print('Check that message received from Alice contains an attachment')
assert receivedJson['object']['attachment']
@ -2913,6 +2915,7 @@ def _testReplyToPublicPost() -> None:
'<a href=\"https://rat.site/@ninjarodent\" ' + \
'class=\"u-url mention\">@<span>ninjarodent</span>' + \
'</a></span> This is a test.</p>'
reply['object']['contentMap'][systemLanguage] = reply['object']['content']
assert reply['object']['tag'][0]['type'] == 'Mention'
assert reply['object']['tag'][0]['name'] == '@ninjarodent@rat.site'
assert reply['object']['tag'][0]['href'] == \
@ -3444,6 +3447,8 @@ def _testLinksWithinPost() -> None:
'rel="nofollow noopener noreferrer" target="_blank">' + \
'<span class="invisible">https://</span>' + \
'<span class="ellipsis">freedombone.net</span></a></p>'
assert postJsonObject['object']['content'] == \
postJsonObject['object']['contentMap'][systemLanguage]
content = "<p>Some text</p><p>Other text</p><p>More text</p>" + \
"<pre><code>Errno::EOHNOES (No such file or rodent @ " + \
@ -3467,6 +3472,7 @@ def _testLinksWithinPost() -> None:
testEventDate, testEventTime, testLocation,
testIsArticle, systemLanguage)
assert postJsonObject['object']['content'] == content
assert postJsonObject['object']['contentMap'][systemLanguage] == content
def _testMastoApi():

37
utils.py
View File

@ -42,6 +42,8 @@ def getContentFromPost(postJsonObject: {}, systemLanguage: str) -> str:
if thisPostJson['contentMap'].get(systemLanguage):
if isinstance(thisPostJson['contentMap'][systemLanguage], str):
return thisPostJson['contentMap'][systemLanguage]
if not isinstance(content, str):
return ''
return content
@ -1732,41 +1734,6 @@ def getCSS(baseDir: str, cssFilename: str, cssCache: {}) -> str:
return None
def isEventPost(messageJson: {}) -> bool:
"""Is the given post a mobilizon-type event activity?
See https://framagit.org/framasoft/mobilizon/-/blob/
master/lib/federation/activity_stream/converter/event.ex
"""
if not messageJson.get('id'):
return False
if not messageJson.get('actor'):
return False
if not hasObjectDict(messageJson):
return False
if not messageJson['object'].get('type'):
return False
if messageJson['object']['type'] != 'Event':
return False
print('Event arriving')
if not messageJson['object'].get('startTime'):
print('No event start time')
return False
if not messageJson['object'].get('actor'):
print('No event actor')
return False
if not messageJson['object'].get('content'):
print('No event content')
return False
if not messageJson['object'].get('name'):
print('No event name')
return False
if not messageJson['object'].get('uuid'):
print('No event UUID')
return False
print('Event detected')
return True
def isBlogPost(postJsonObject: {}) -> bool:
"""Is the given post a blog post?
"""

6
webapp_column_right.py
View File

@ -11,6 +11,7 @@ import os
from datetime import datetime
from content import removeLongWords
from content import limitRepeatedWords
from utils import getContentFromPost
from utils import removeHtml
from utils import locatePost
from utils import loadJson
@ -636,7 +637,8 @@ def htmlEditNewswire(cssCache: {}, translate: {}, baseDir: str, path: str,
def htmlEditNewsPost(cssCache: {}, translate: {}, baseDir: str, path: str,
domain: str, port: int,
httpPrefix: str, postUrl: str) -> str:
httpPrefix: str, postUrl: str,
systemLanguage: str) -> str:
"""Edits a news post on the news/features timeline
"""
if '/users/' not in path:
@ -696,7 +698,7 @@ def htmlEditNewsPost(cssCache: {}, translate: {}, baseDir: str, path: str,
' <input type="text" name="newsPostTitle" value="' + \
newsPostTitle + '"><br>\n'
newsPostContent = postJsonObject['object']['content']
newsPostContent = getContentFromPost(postJsonObject, systemLanguage)
editNewsPostForm += \
' <textarea id="message" name="editedNewsPost" ' + \
'style="height:600px" spellcheck="true">' + \

5
webapp_moderation.py
View File

@ -69,7 +69,8 @@ def htmlModeration(cssCache: {}, defaultTimeline: str,
def htmlAccountInfo(cssCache: {}, translate: {},
baseDir: str, httpPrefix: str,
nickname: str, domain: str, port: int,
searchHandle: str, debug: bool) -> str:
searchHandle: str, debug: bool,
systemLanguage: str) -> str:
"""Shows which domains a search handle interacts with.
This screen is shown if a moderator enters a handle and selects info
on the moderation screen
@ -111,7 +112,7 @@ def htmlAccountInfo(cssCache: {}, translate: {},
baseDir, searchNickname, searchDomain,
proxyType, searchPort,
httpPrefix, debug,
__version__, wordFrequency)
__version__, wordFrequency, systemLanguage)
# get a list of any blocked followers
followersList = \

14
webapp_post.py
View File

@ -45,7 +45,6 @@ from utils import updateRecentPostsCache
from utils import removeIdEnding
from utils import getNicknameFromActor
from utils import getDomainFromActor
from utils import isEventPost
from utils import acctDir
from content import limitRepeatedWords
from content import replaceEmojiFromTags
@ -1310,7 +1309,8 @@ def individualPostAsHtml(allowDownloads: bool,
projectVersion, translate,
YTReplacementDomain,
allowLocalNetworkAccess,
recentPostsCache, False)
recentPostsCache, False,
systemLanguage)
if not postJsonAnnounce:
# if the announce could not be downloaded then mark it as rejected
rejectPostId(baseDir, nickname, domain, postJsonObject['id'],
@ -1415,13 +1415,11 @@ def individualPostAsHtml(allowDownloads: bool,
_logPostTiming(enableTimingLog, postStartTime, '10')
isEvent = isEventPost(postJsonObject)
_logPostTiming(enableTimingLog, postStartTime, '11')
editStr = _getEditIconHtml(baseDir, nickname, domainFull,
postJsonObject, actorNickname,
translate, isEvent)
translate, False)
_logPostTiming(enableTimingLog, postStartTime, '11')
announceStr = \
_getAnnounceIconHtml(isAnnounced,
@ -1588,6 +1586,8 @@ def individualPostAsHtml(allowDownloads: bool,
if postJsonObject['object'].get('cipherText'):
postJsonObject['object']['content'] = \
E2EEdecryptMessageFromDevice(postJsonObject['object'])
postJsonObject['object']['contentMap'][systemLanguage] = \
postJsonObject['object']['content']
contentStr = getContentFromPost(postJsonObject, systemLanguage)
if not contentStr:

6
webapp_search.py
View File

@ -11,6 +11,7 @@ import os
from shutil import copyfile
import urllib.parse
from datetime import datetime
from utils import getContentFromPost
from utils import isAccountDir
from utils import getConfigParam
from utils import getFullDomain
@ -821,7 +822,7 @@ def rssHashtagSearch(nickname: str, domain: str, port: int,
postsPerPage: int,
session, cachedWebfingers: {}, personCache: {},
httpPrefix: str, projectVersion: str,
YTReplacementDomain: str) -> str:
YTReplacementDomain: str, systemLanguage: str) -> str:
"""Show an rss feed for a hashtag
"""
if hashtag.startswith('#'):
@ -902,7 +903,8 @@ def rssHashtagSearch(nickname: str, domain: str, port: int,
' <title>' + \
postJsonObject['object']['summary'] + \
'</title>'
description = postJsonObject['object']['content']
description = \
getContentFromPost(postJsonObject, systemLanguage)
description = firstParagraphFromString(description)
hashtagFeed += \
' <description>' + description + '</description>'