From 2e6fe854f8d09881a0e26898dd6f11682720efdc Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 9 Feb 2021 22:29:57 +0000 Subject: [PATCH 01/15] More checks --- content.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content.py b/content.py index 1d8334412..15387c926 100644 --- a/content.py +++ b/content.py @@ -389,6 +389,8 @@ def validHashTag(hashtag: str) -> bool: 'ŴŵÝýŸÿŶŷŹźŽžŻż') if set(hashtag).issubset(validChars): return True + if '#' in hashtag or '"' in hashtag or '&' in hashtag: + return False if isValidLanguage(hashtag): return True return False From 08263266539a6ff41b1d1797e827384541dd386b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 9 Feb 2021 22:42:29 +0000 Subject: [PATCH 02/15] Validation of categories --- content.py | 2 -- webapp_hashtagswarm.py | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/content.py b/content.py index 15387c926..1d8334412 100644 --- a/content.py +++ b/content.py @@ -389,8 +389,6 @@ def validHashTag(hashtag: str) -> bool: 'ŴŵÝýŸÿŶŷŹźŽžŻż') if set(hashtag).issubset(validChars): return True - if '#' in hashtag or '"' in hashtag or '&' in hashtag: - return False if isValidLanguage(hashtag): return True return False diff --git a/webapp_hashtagswarm.py b/webapp_hashtagswarm.py index 19f08661f..f2ffb577d 100644 --- a/webapp_hashtagswarm.py +++ b/webapp_hashtagswarm.py @@ -203,8 +203,12 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str: categoryStr = \ getHashtagCategory(baseDir, hashTagName) if len(categoryStr) < maxTagLength: - if categoryStr not in categorySwarm: - categorySwarm.append(categoryStr) + if '#' not in categoryStr and \ + '&' not in categoryStr and \ + '"' not in categoryStr and \ + "'" not in categoryStr: + if categoryStr not in categorySwarm: + categorySwarm.append(categoryStr) break break From b16fb0d24cfc4352d358abc310a14fc2a4e2b052 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 11:24:34 +0000 Subject: [PATCH 03/15] Improve checking of site active status This uses a defluffed version of webchk --- posts.py | 2 +- siteactive.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests.py | 6 ++- utils.py | 25 ----------- 4 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 siteactive.py diff --git a/posts.py b/posts.py index d44257bd2..9e94ba54e 100644 --- a/posts.py +++ b/posts.py @@ -30,6 +30,7 @@ from session import postJsonString from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader +from siteactive import siteIsActive from utils import fileLastModified from utils import isPublicPost from utils import hasUsersPath @@ -38,7 +39,6 @@ from utils import getFullDomain from utils import getFollowersList from utils import isEvil from utils import removeIdEnding -from utils import siteIsActive from utils import getCachedPostFilename from utils import getStatusNumber from utils import createPersonDir diff --git a/siteactive.py b/siteactive.py new file mode 100644 index 000000000..ca530bf49 --- /dev/null +++ b/siteactive.py @@ -0,0 +1,121 @@ +__filename__ = "siteactive.py" +__author__ = "Bob Mottram" +__credits__ = ["webchk"] +__license__ = "AGPL3+" +__version__ = "1.2.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import http.client +from urllib.parse import urlparse +import ssl + + +class Result: + """Holds result of an URL check. + + The redirect attribute is a Result object that the URL was redirected to. + + The sitemap_urls attribute will contain a list of Result object if url + is a sitemap file and http_response() was run with parse set to True. + """ + def __init__(self, url): + self.url = url + self.status = 0 + self.desc = '' + self.headers = None + self.latency = 0 + self.content = '' + self.redirect = None + self.sitemap_urls = None + + def __repr__(self): + if self.status == 0: + return '{} ... {}'.format(self.url, self.desc) + return '{} ... {} {} ({})'.format( + self.url, self.status, self.desc, self.latency + ) + + def fill_headers(self, headers): + """Takes a list of tuples and converts it a dictionary.""" + self.headers = {h[0]: h[1] for h in headers} + + +def _siteActiveParseUrl(url): + """Returns an object with properties representing + + scheme: URL scheme specifier + netloc: Network location part + path: Hierarchical path + params: Parameters for last path element + query: Query component + fragment: Fragment identifier + username: User name + password: Password + hostname: Host name (lower case) + port: Port number as integer, if present + """ + loc = urlparse(url) + + # if the scheme (http, https ...) is not available urlparse wont work + if loc.scheme == "": + url = "http://" + url + loc = urlparse(url) + return loc + + +def _siteACtiveHttpConnect(loc, timeout: int): + """Connects to the host and returns an HTTP or HTTPS connections.""" + if loc.scheme == "https": + ssl_context = ssl.SSLContext() + return http.client.HTTPSConnection( + loc.netloc, context=ssl_context, timeout=timeout) + return http.client.HTTPConnection(loc.netloc, timeout=timeout) + + +def _siteActiveHttpRequest(loc, timeout: int): + """Performs a HTTP request and return response in a Result object. + """ + conn = _siteACtiveHttpConnect(loc, timeout) + method = 'HEAD' + + conn.request(method, loc.path) + resp = conn.getresponse() + + result = Result(loc.geturl()) + result.status = resp.status + result.desc = resp.reason + result.fill_headers(resp.getheaders()) + + conn.close() + return result + + +def siteIsActive(url: str, timeout=10) -> bool: + """Returns true if the current url is resolvable. + This can be used to check that an instance is online before + trying to send posts to it. + """ + if not url.startswith('http'): + return False + if '.onion/' in url or '.i2p/' in url or \ + url.endswith('.onion') or \ + url.endswith('.i2p'): + # skip this check for onion and i2p + return True + + loc = _siteActiveParseUrl(url) + result = Result(url=url) + + try: + result = _siteActiveHttpRequest(loc, timeout) + + if 400 <= result.status < 500: + return result + + return True + + except BaseException: + pass + return False diff --git a/tests.py b/tests.py index 308e1fb0c..7186012a1 100644 --- a/tests.py +++ b/tests.py @@ -38,7 +38,7 @@ from utils import getFullDomain from utils import validNickname from utils import firstParagraphFromString from utils import removeIdEnding -from utils import siteIsActive +from siteactive import siteIsActive from utils import updateRecentPostsCache from utils import followPerson from utils import getNicknameFromActor @@ -2067,6 +2067,7 @@ def testJsonld(): def testSiteIsActive(): print('testSiteIsActive') + assert(siteIsActive('https://archive.org')) assert(siteIsActive('https://mastodon.social')) assert(not siteIsActive('https://notarealwebsite.a.b.c')) @@ -2818,7 +2819,8 @@ def testFunctions(): 'createServerBob', 'createServerEve', 'E2EEremoveDevice', - 'setOrganizationScheme' + 'setOrganizationScheme', + 'fill_headers' ] excludeImports = [ 'link', diff --git a/utils.py b/utils.py index 0f3d811cf..6e9b82e2b 100644 --- a/utils.py +++ b/utils.py @@ -11,9 +11,6 @@ import time import shutil import datetime import json -from socket import error as SocketError -import errno -import urllib.request import idna from pprint import pprint from calendar import monthrange @@ -1841,28 +1838,6 @@ def updateAnnounceCollection(recentPostsCache: {}, saveJson(postJsonObject, postFilename) -def siteIsActive(url: str) -> bool: - """Returns true if the current url is resolvable. - This can be used to check that an instance is online before - trying to send posts to it. - """ - if not url.startswith('http'): - return False - if '.onion/' in url or '.i2p/' in url or \ - url.endswith('.onion') or \ - url.endswith('.i2p'): - # skip this check for onion and i2p - return True - try: - req = urllib.request.Request(url) - urllib.request.urlopen(req, timeout=10) # nosec - return True - except SocketError as e: - if e.errno == errno.ECONNRESET: - print('WARN: connection was reset during siteIsActive') - return False - - def weekDayOfMonthStart(monthNumber: int, year: int) -> int: """Gets the day number of the first day of the month 1=sun, 7=sat From daff4521225054916cfff1ab642baa31617b750e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 12:44:58 +0000 Subject: [PATCH 04/15] default theme post separator --- theme/default/icons/separator_right.png | Bin 0 -> 1272 bytes theme/default/theme.json | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 theme/default/icons/separator_right.png diff --git a/theme/default/icons/separator_right.png b/theme/default/icons/separator_right.png new file mode 100644 index 0000000000000000000000000000000000000000..3f31f0fce4a0af81811c5785648734030ae63f70 GIT binary patch literal 1272 zcmV zaB^>EX>4U6ba`-PAZ2)IW&i+q+O3sqmgFW3h5xgPSpt#}63f9n&dCn4{JCJNx@x+U z%p>L)Q$h&&eliut-+#^U4Hv~2LsIixa>h8Ok_uN0Jl}46va4W{PTS4}i-TX|5kz3-XD+n&EqlL4j-5Nfl?lNJOa8Hif1CW37UpbABHDuT z)(Uy?5*%qPa;t|IfY3bLbPN1w2Y-AiER-q;>XzB@f;C=uh)e0Cwq$b__$i+ES3$x0 z2>^+(w;~yoeWI92lrrJmU?YN#4nP%|vn0MifK<67Nlqe}gR%448*7ZVFDqwWW}Hm` zp_0YGrbq>>loa)2$x%a9MU$##HT9rHYnGg{=A13BL#~=wGPP`GZpEsLCs)sI?q0kW zE`l>qODfM;UdZ)~3ufb>?Z)XPNa$?NI&Het{YtYP?A;KYLPxHJCj@&^k_Z zF#|D91md;`prCm%i%u!=BDa{u!uTkZk)$p*q0=G;glQ1#q$j%%azDi_sQ(l<{x5Q2 zq5D6`1)%%P?F(vs{m->c?EVUurcvL4<-TiYTFi2<8a3PgKDO<~3E+PQ zuJo3_TnA=8Nw2lE@Db3r4P0EeG-VIC+yMrj4B3<&$xkKZa=`l;eNzVLzXiJ2+_^Q+ zarywHXjX|E;NTD#%~STe$Gf}Q=l1WN=KOvD+w*eK0y}b;00006VoOIv00000008+z zyMF)x010qNS#tmY3ljhU3ljkVnw%H_000McNliru}OFbbz+eQN*!01-( Date: Wed, 10 Feb 2021 13:16:24 +0000 Subject: [PATCH 05/15] Only accept new blog posts if they have a title --- daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon.py b/daemon.py index 7afb725b9..fd1f18111 100644 --- a/daemon.py +++ b/daemon.py @@ -12393,6 +12393,9 @@ class PubServer(BaseHTTPRequestHandler): else: return -1 elif postType == 'newblog': + if not fields['subject']: + print('WARN: blog posts must have a title') + return -1 # citations button on newblog screen if citationsButtonPress: messageJson = \ From 0a110e2aaab9eb2d0de9464ea43e315074d2585d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 13:17:19 +0000 Subject: [PATCH 06/15] Only accept new blog posts if they have content --- daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon.py b/daemon.py index fd1f18111..2345be05b 100644 --- a/daemon.py +++ b/daemon.py @@ -12396,6 +12396,9 @@ class PubServer(BaseHTTPRequestHandler): if not fields['subject']: print('WARN: blog posts must have a title') return -1 + if not fields['message']: + print('WARN: blog posts must have content') + return -1 # citations button on newblog screen if citationsButtonPress: messageJson = \ From 57fcdcca3898c626a878b1092af23cb8e3e3c00c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 13:31:19 +0000 Subject: [PATCH 07/15] Recalculate the newswire if a new blog post is created --- daemon.py | 2 ++ newsdaemon.py | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 2345be05b..e24e3d8a9 100644 --- a/daemon.py +++ b/daemon.py @@ -255,6 +255,7 @@ from newswire import rss2Footer from newswire import loadHashtagCategories from newsdaemon import runNewswireWatchdog from newsdaemon import runNewswireDaemon +from newsdaemon import refreshNewswire from filters import isFiltered from filters import addGlobalFilter from filters import removeGlobalFilter @@ -12444,6 +12445,7 @@ class PubServer(BaseHTTPRequestHandler): if fields['schedulePost']: return 1 if self._postToOutbox(messageJson, __version__, nickname): + refreshNewswire(self.server.baseDir) populateReplies(self.server.baseDir, self.server.httpPrefix, self.server.domainFull, diff --git a/newsdaemon.py b/newsdaemon.py index 84f2f4bc4..e52932072 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -660,6 +660,7 @@ def runNewswireDaemon(baseDir: str, httpd, """Periodically updates RSS feeds """ newswireStateFilename = baseDir + '/accounts/.newswirestate.json' + refreshFilename = baseDir + '/accounts/.refresh_newswire' # initial sleep to allow the system to start up time.sleep(50) @@ -722,7 +723,16 @@ def runNewswireDaemon(baseDir: str, httpd, httpd.maxNewsPosts) # wait a while before the next feeds update - time.sleep(1200) + for tick in range(120): + time.sleep(10) + # if a new blog post has been created then stop + # waiting and recalculate the newswire + if os.path.isfile(refreshFilename): + try: + os.remove(refreshFilename) + except BaseException: + pass + break def runNewswireWatchdog(projectVersion: str, httpd) -> None: @@ -740,3 +750,15 @@ def runNewswireWatchdog(projectVersion: str, httpd) -> None: newswireOriginal.clone(runNewswireDaemon) httpd.thrNewswireDaemon.start() print('Restarting newswire daemon...') + + +def refreshNewswire(baseDir: str) -> None: + """Causes the newswire to be updated. + This creates a file which is then detected by the daemon + """ + refreshFilename = baseDir + '/accounts/.refresh_newswire' + if os.path.isfile(refreshFilename): + return + refreshFile = open(refreshFilename, 'w+') + refreshFile.write('\n') + refreshFile.close() From 41045ba6735b7db54a8b91a1fab0f71687f65a83 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 14:29:52 +0000 Subject: [PATCH 08/15] Check later --- daemon.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/daemon.py b/daemon.py index e24e3d8a9..378006b80 100644 --- a/daemon.py +++ b/daemon.py @@ -12394,12 +12394,6 @@ class PubServer(BaseHTTPRequestHandler): else: return -1 elif postType == 'newblog': - if not fields['subject']: - print('WARN: blog posts must have a title') - return -1 - if not fields['message']: - print('WARN: blog posts must have content') - return -1 # citations button on newblog screen if citationsButtonPress: messageJson = \ @@ -12426,6 +12420,12 @@ class PubServer(BaseHTTPRequestHandler): return 1 else: return -1 + if not fields['subject']: + print('WARN: blog posts must have a title') + return -1 + if not fields['message']: + print('WARN: blog posts must have content') + return -1 # submit button on newblog screen messageJson = \ createBlogPost(self.server.baseDir, nickname, From 5d54d0131f6c869b4271a4d3770d4261b8ccb3a8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 15:23:51 +0000 Subject: [PATCH 09/15] Preceding slash --- webapp_post.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp_post.py b/webapp_post.py index dd4c9a24b..c9505c819 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -327,9 +327,9 @@ def _getEditIconHtml(baseDir: str, nickname: str, domainFull: str, """ editStr = '' actor = postJsonObject['actor'] - if (actor.endswith(domainFull + '/users/' + nickname) or + if (actor.endswith('/' + domainFull + '/users/' + nickname) or (isEditor(baseDir, nickname) and - actor.endswith(domainFull + '/users/news'))): + actor.endswith('/' + domainFull + '/users/news'))): postId = postJsonObject['object']['id'] From 1f124e9b5e6b97540873e7c1c384f1120c0fd913 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 17:03:51 +0000 Subject: [PATCH 10/15] Set new blog posts as article type --- daemon.py | 4 ++-- posts.py | 28 ++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/daemon.py b/daemon.py index 378006b80..3e091bebc 100644 --- a/daemon.py +++ b/daemon.py @@ -393,7 +393,7 @@ class PubServer(BaseHTTPRequestHandler): schedulePost, eventDate, eventTime, - location) + location, False) if messageJson: # name field contains the answer messageJson['object']['name'] = answer @@ -12374,7 +12374,7 @@ class PubServer(BaseHTTPRequestHandler): fields['replyTo'], fields['replyTo'], fields['subject'], fields['schedulePost'], fields['eventDate'], fields['eventTime'], - fields['location']) + fields['location'], False) if messageJson: if fields['schedulePost']: return 1 diff --git a/posts.py b/posts.py index 9e94ba54e..4e3c2dc5e 100644 --- a/posts.py +++ b/posts.py @@ -1389,10 +1389,22 @@ def createPublicPost(baseDir: str, imageDescription: str, inReplyTo=None, inReplyToAtomUri=None, subject=None, schedulePost=False, - eventDate=None, eventTime=None, location=None) -> {}: + eventDate=None, eventTime=None, location=None, + isArticle=False) -> {}: """Public post """ domainFull = getFullDomain(domain, port) + isModerationReport = False + eventUUID = None + category = None + joinMode = None + endDate = None + endTime = None + maximumAttendeeCapacity = None + repliesModerationOption = None + anonymousParticipationEnabled = None + eventStatus = None + ticketUrl = None return _createPostBase(baseDir, nickname, domain, port, 'https://www.w3.org/ns/activitystreams#Public', httpPrefix + '://' + domainFull + '/users/' + @@ -1401,10 +1413,14 @@ def createPublicPost(baseDir: str, clientToServer, commentsEnabled, attachImageFilename, mediaType, imageDescription, - False, False, inReplyTo, inReplyToAtomUri, subject, + isModerationReport, isArticle, + inReplyTo, inReplyToAtomUri, subject, schedulePost, eventDate, eventTime, location, - None, None, None, None, None, - None, None, None, None, None) + eventUUID, category, joinMode, endDate, endTime, + maximumAttendeeCapacity, + repliesModerationOption, + anonymousParticipationEnabled, + eventStatus, ticketUrl) def createBlogPost(baseDir: str, @@ -1425,7 +1441,7 @@ def createBlogPost(baseDir: str, imageDescription, inReplyTo, inReplyToAtomUri, subject, schedulePost, - eventDate, eventTime, location) + eventDate, eventTime, location, True) blog['object']['type'] = 'Article' # append citations tags, stored in a file @@ -1477,7 +1493,7 @@ def createNewsPost(baseDir: str, imageDescription, inReplyTo, inReplyToAtomUri, subject, schedulePost, - eventDate, eventTime, location) + eventDate, eventTime, location, True) blog['object']['type'] = 'Article' return blog From 2f5d270c3f2d25bda71f1dd063e63a03fd5bb0d7 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 10 Feb 2021 18:13:41 +0000 Subject: [PATCH 11/15] Set article type --- posts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posts.py b/posts.py index 4e3c2dc5e..df27ed6f8 100644 --- a/posts.py +++ b/posts.py @@ -1047,6 +1047,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, postObjectType = 'Note' if eventUUID: postObjectType = 'Event' + if isArticle: + postObjectType = 'Article' if not clientToServer: actorUrl = httpPrefix + '://' + domain + '/users/' + nickname @@ -1442,7 +1444,6 @@ def createBlogPost(baseDir: str, inReplyTo, inReplyToAtomUri, subject, schedulePost, eventDate, eventTime, location, True) - blog['object']['type'] = 'Article' # append citations tags, stored in a file citationsFilename = \ From 2cbcd4eeb27aa51e796a1ac1cdd6d2c6679a59af Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 11 Feb 2021 10:01:27 +0000 Subject: [PATCH 12/15] Tidying --- posts.py | 59 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/posts.py b/posts.py index df27ed6f8..d64397055 100644 --- a/posts.py +++ b/posts.py @@ -1425,6 +1425,37 @@ def createPublicPost(baseDir: str, eventStatus, ticketUrl) +def _appendCitationsToBlogPost(baseDir: str, + nickname: str, domain: str, + blogJson: {}) -> None: + """Appends any citations to a new blog post + """ + # append citations tags, stored in a file + citationsFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/.citations.txt' + if not os.path.isfile(citationsFilename): + return + citationsSeparator = '#####' + with open(citationsFilename, "r") as f: + citations = f.readlines() + for line in citations: + if citationsSeparator not in line: + continue + sections = line.strip().split(citationsSeparator) + if len(sections) != 3: + continue + # dateStr = sections[0] + title = sections[1] + link = sections[2] + tagJson = { + "type": "Article", + "name": title, + "url": link + } + blogJson['object']['tag'].append(tagJson) + + def createBlogPost(baseDir: str, nickname: str, domain: str, port: int, httpPrefix: str, content: str, followersOnly: bool, saveToFile: bool, @@ -1434,7 +1465,7 @@ def createBlogPost(baseDir: str, inReplyTo=None, inReplyToAtomUri=None, subject=None, schedulePost=False, eventDate=None, eventTime=None, location=None) -> {}: - blog = \ + blogJson = \ createPublicPost(baseDir, nickname, domain, port, httpPrefix, content, followersOnly, saveToFile, @@ -1445,31 +1476,9 @@ def createBlogPost(baseDir: str, schedulePost, eventDate, eventTime, location, True) - # append citations tags, stored in a file - citationsFilename = \ - baseDir + '/accounts/' + \ - nickname + '@' + domain + '/.citations.txt' - if os.path.isfile(citationsFilename): - citationsSeparator = '#####' - with open(citationsFilename, "r") as f: - citations = f.readlines() - for line in citations: - if citationsSeparator not in line: - continue - sections = line.strip().split(citationsSeparator) - if len(sections) != 3: - continue - # dateStr = sections[0] - title = sections[1] - link = sections[2] - tagJson = { - "type": "Article", - "name": title, - "url": link - } - blog['object']['tag'].append(tagJson) + _appendCitationsToBlogPost(baseDir, nickname, domain, blogJson) - return blog + return blogJson def createNewsPost(baseDir: str, From 6806218202cd2e760bf5e12d89368ca1c603095e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 11 Feb 2021 10:33:56 +0000 Subject: [PATCH 13/15] Screen for outgoing abusive posts --- posts.py | 5 ++++- utils.py | 23 ++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/posts.py b/posts.py index d64397055..d9695ada9 100644 --- a/posts.py +++ b/posts.py @@ -31,6 +31,7 @@ from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader from siteactive import siteIsActive +from utils import removeInvalidCharacters from utils import fileLastModified from utils import isPublicPost from utils import hasUsersPath @@ -823,7 +824,7 @@ def validContentWarning(cw: str) -> str: # so remove them if '#' in cw: cw = cw.replace('#', '').replace(' ', ' ') - return cw + return removeInvalidCharacters(cw) def _loadAutoCW(baseDir: str, nickname: str, domain: str) -> []: @@ -880,6 +881,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, eventStatus=None, ticketUrl=None) -> {}: """Creates a message """ + content = removeInvalidCharacters(content) + subject = _addAutoCW(baseDir, nickname, domain, subject, content) if nickname != 'news': diff --git a/utils.py b/utils.py index 6e9b82e2b..2755cb846 100644 --- a/utils.py +++ b/utils.py @@ -18,6 +18,13 @@ from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes +# posts containing these strings will always get screened out, +# both incoming and outgoing. +# Could include dubious clacks or admin dogwhistles +invalidCharacters = ( + '卐', '卍', '࿕', '࿖', '࿗', '࿘' +) + def getSHA256(msg: str): """Returns a SHA256 hash of the given string @@ -514,17 +521,23 @@ def isEvil(domain: str) -> bool: def containsInvalidChars(jsonStr: str) -> bool: """Does the given json string contain invalid characters? - e.g. dubious clacks/admin dogwhistles """ - invalidStrings = { - '卐', '卍', '࿕', '࿖', '࿗', '࿘' - } - for isInvalid in invalidStrings: + for isInvalid in invalidCharacters: if isInvalid in jsonStr: return True return False +def removeInvalidCharacters(text: str) -> str: + """Removes any invalid characters from a string + """ + for isInvalid in invalidCharacters: + if isInvalid not in text: + continue + text = text.replace(isInvalid, '') + return text + + def createPersonDir(nickname: str, domain: str, baseDir: str, dirname: str) -> str: """Create a directory for a person From 9ddad5366b1004f2a4bfa1904baf1efc14f89740 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 11 Feb 2021 10:58:51 +0000 Subject: [PATCH 14/15] Remove invalid characters from outgoing content warnings --- posts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posts.py b/posts.py index d9695ada9..c8ccf1025 100644 --- a/posts.py +++ b/posts.py @@ -927,7 +927,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, sensitive = False summary = None if subject: - summary = validContentWarning(subject) + summary = removeInvalidCharacters(validContentWarning(subject)) sensitive = True toRecipients = [] From 292fa6fd8f08957be5e0b18e90637882d8aefd67 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 11 Feb 2021 11:02:05 +0000 Subject: [PATCH 15/15] Abbreviate function name --- posts.py | 8 ++++---- utils.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/posts.py b/posts.py index c8ccf1025..fa0d72452 100644 --- a/posts.py +++ b/posts.py @@ -31,7 +31,7 @@ from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader from siteactive import siteIsActive -from utils import removeInvalidCharacters +from utils import removeInvalidChars from utils import fileLastModified from utils import isPublicPost from utils import hasUsersPath @@ -824,7 +824,7 @@ def validContentWarning(cw: str) -> str: # so remove them if '#' in cw: cw = cw.replace('#', '').replace(' ', ' ') - return removeInvalidCharacters(cw) + return removeInvalidChars(cw) def _loadAutoCW(baseDir: str, nickname: str, domain: str) -> []: @@ -881,7 +881,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, eventStatus=None, ticketUrl=None) -> {}: """Creates a message """ - content = removeInvalidCharacters(content) + content = removeInvalidChars(content) subject = _addAutoCW(baseDir, nickname, domain, subject, content) @@ -927,7 +927,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, sensitive = False summary = None if subject: - summary = removeInvalidCharacters(validContentWarning(subject)) + summary = removeInvalidChars(validContentWarning(subject)) sensitive = True toRecipients = [] diff --git a/utils.py b/utils.py index 2755cb846..5a58d45ce 100644 --- a/utils.py +++ b/utils.py @@ -528,7 +528,7 @@ def containsInvalidChars(jsonStr: str) -> bool: return False -def removeInvalidCharacters(text: str) -> str: +def removeInvalidChars(text: str) -> str: """Removes any invalid characters from a string """ for isInvalid in invalidCharacters: