diff --git a/daemon.py b/daemon.py index 7afb725b9..3e091bebc 100644 --- a/daemon.py +++ b/daemon.py @@ -255,6 +255,7 @@ from newswire import rss2Footer from newswire import loadHashtagCategories from newsdaemon import runNewswireWatchdog from newsdaemon import runNewswireDaemon +from newsdaemon import refreshNewswire from filters import isFiltered from filters import addGlobalFilter from filters import removeGlobalFilter @@ -392,7 +393,7 @@ class PubServer(BaseHTTPRequestHandler): schedulePost, eventDate, eventTime, - location) + location, False) if messageJson: # name field contains the answer messageJson['object']['name'] = answer @@ -12373,7 +12374,7 @@ class PubServer(BaseHTTPRequestHandler): fields['replyTo'], fields['replyTo'], fields['subject'], fields['schedulePost'], fields['eventDate'], fields['eventTime'], - fields['location']) + fields['location'], False) if messageJson: if fields['schedulePost']: return 1 @@ -12419,6 +12420,12 @@ class PubServer(BaseHTTPRequestHandler): return 1 else: return -1 + if not fields['subject']: + print('WARN: blog posts must have a title') + return -1 + if not fields['message']: + print('WARN: blog posts must have content') + return -1 # submit button on newblog screen messageJson = \ createBlogPost(self.server.baseDir, nickname, @@ -12438,6 +12445,7 @@ class PubServer(BaseHTTPRequestHandler): if fields['schedulePost']: return 1 if self._postToOutbox(messageJson, __version__, nickname): + refreshNewswire(self.server.baseDir) populateReplies(self.server.baseDir, self.server.httpPrefix, self.server.domainFull, diff --git a/newsdaemon.py b/newsdaemon.py index 84f2f4bc4..e52932072 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -660,6 +660,7 @@ def runNewswireDaemon(baseDir: str, httpd, """Periodically updates RSS feeds """ newswireStateFilename = baseDir + '/accounts/.newswirestate.json' + refreshFilename = baseDir + '/accounts/.refresh_newswire' # initial sleep to allow the system to start up time.sleep(50) @@ -722,7 +723,16 @@ def runNewswireDaemon(baseDir: str, httpd, httpd.maxNewsPosts) # wait a while before the next feeds update - time.sleep(1200) + for tick in range(120): + time.sleep(10) + # if a new blog post has been created then stop + # waiting and recalculate the newswire + if os.path.isfile(refreshFilename): + try: + os.remove(refreshFilename) + except BaseException: + pass + break def runNewswireWatchdog(projectVersion: str, httpd) -> None: @@ -740,3 +750,15 @@ def runNewswireWatchdog(projectVersion: str, httpd) -> None: newswireOriginal.clone(runNewswireDaemon) httpd.thrNewswireDaemon.start() print('Restarting newswire daemon...') + + +def refreshNewswire(baseDir: str) -> None: + """Causes the newswire to be updated. + This creates a file which is then detected by the daemon + """ + refreshFilename = baseDir + '/accounts/.refresh_newswire' + if os.path.isfile(refreshFilename): + return + refreshFile = open(refreshFilename, 'w+') + refreshFile.write('\n') + refreshFile.close() diff --git a/posts.py b/posts.py index d44257bd2..fa0d72452 100644 --- a/posts.py +++ b/posts.py @@ -30,6 +30,8 @@ from session import postJsonString from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader +from siteactive import siteIsActive +from utils import removeInvalidChars from utils import fileLastModified from utils import isPublicPost from utils import hasUsersPath @@ -38,7 +40,6 @@ from utils import getFullDomain from utils import getFollowersList from utils import isEvil from utils import removeIdEnding -from utils import siteIsActive from utils import getCachedPostFilename from utils import getStatusNumber from utils import createPersonDir @@ -823,7 +824,7 @@ def validContentWarning(cw: str) -> str: # so remove them if '#' in cw: cw = cw.replace('#', '').replace(' ', ' ') - return cw + return removeInvalidChars(cw) def _loadAutoCW(baseDir: str, nickname: str, domain: str) -> []: @@ -880,6 +881,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, eventStatus=None, ticketUrl=None) -> {}: """Creates a message """ + content = removeInvalidChars(content) + subject = _addAutoCW(baseDir, nickname, domain, subject, content) if nickname != 'news': @@ -924,7 +927,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, sensitive = False summary = None if subject: - summary = validContentWarning(subject) + summary = removeInvalidChars(validContentWarning(subject)) sensitive = True toRecipients = [] @@ -1047,6 +1050,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, postObjectType = 'Note' if eventUUID: postObjectType = 'Event' + if isArticle: + postObjectType = 'Article' if not clientToServer: actorUrl = httpPrefix + '://' + domain + '/users/' + nickname @@ -1389,10 +1394,22 @@ def createPublicPost(baseDir: str, imageDescription: str, inReplyTo=None, inReplyToAtomUri=None, subject=None, schedulePost=False, - eventDate=None, eventTime=None, location=None) -> {}: + eventDate=None, eventTime=None, location=None, + isArticle=False) -> {}: """Public post """ domainFull = getFullDomain(domain, port) + isModerationReport = False + eventUUID = None + category = None + joinMode = None + endDate = None + endTime = None + maximumAttendeeCapacity = None + repliesModerationOption = None + anonymousParticipationEnabled = None + eventStatus = None + ticketUrl = None return _createPostBase(baseDir, nickname, domain, port, 'https://www.w3.org/ns/activitystreams#Public', httpPrefix + '://' + domainFull + '/users/' + @@ -1401,10 +1418,45 @@ def createPublicPost(baseDir: str, clientToServer, commentsEnabled, attachImageFilename, mediaType, imageDescription, - False, False, inReplyTo, inReplyToAtomUri, subject, + isModerationReport, isArticle, + inReplyTo, inReplyToAtomUri, subject, schedulePost, eventDate, eventTime, location, - None, None, None, None, None, - None, None, None, None, None) + eventUUID, category, joinMode, endDate, endTime, + maximumAttendeeCapacity, + repliesModerationOption, + anonymousParticipationEnabled, + eventStatus, ticketUrl) + + +def _appendCitationsToBlogPost(baseDir: str, + nickname: str, domain: str, + blogJson: {}) -> None: + """Appends any citations to a new blog post + """ + # append citations tags, stored in a file + citationsFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/.citations.txt' + if not os.path.isfile(citationsFilename): + return + citationsSeparator = '#####' + with open(citationsFilename, "r") as f: + citations = f.readlines() + for line in citations: + if citationsSeparator not in line: + continue + sections = line.strip().split(citationsSeparator) + if len(sections) != 3: + continue + # dateStr = sections[0] + title = sections[1] + link = sections[2] + tagJson = { + "type": "Article", + "name": title, + "url": link + } + blogJson['object']['tag'].append(tagJson) def createBlogPost(baseDir: str, @@ -1416,7 +1468,7 @@ def createBlogPost(baseDir: str, inReplyTo=None, inReplyToAtomUri=None, subject=None, schedulePost=False, eventDate=None, eventTime=None, location=None) -> {}: - blog = \ + blogJson = \ createPublicPost(baseDir, nickname, domain, port, httpPrefix, content, followersOnly, saveToFile, @@ -1425,34 +1477,11 @@ def createBlogPost(baseDir: str, imageDescription, inReplyTo, inReplyToAtomUri, subject, schedulePost, - eventDate, eventTime, location) - blog['object']['type'] = 'Article' + eventDate, eventTime, location, True) - # append citations tags, stored in a file - citationsFilename = \ - baseDir + '/accounts/' + \ - nickname + '@' + domain + '/.citations.txt' - if os.path.isfile(citationsFilename): - citationsSeparator = '#####' - with open(citationsFilename, "r") as f: - citations = f.readlines() - for line in citations: - if citationsSeparator not in line: - continue - sections = line.strip().split(citationsSeparator) - if len(sections) != 3: - continue - # dateStr = sections[0] - title = sections[1] - link = sections[2] - tagJson = { - "type": "Article", - "name": title, - "url": link - } - blog['object']['tag'].append(tagJson) + _appendCitationsToBlogPost(baseDir, nickname, domain, blogJson) - return blog + return blogJson def createNewsPost(baseDir: str, @@ -1477,7 +1506,7 @@ def createNewsPost(baseDir: str, imageDescription, inReplyTo, inReplyToAtomUri, subject, schedulePost, - eventDate, eventTime, location) + eventDate, eventTime, location, True) blog['object']['type'] = 'Article' return blog diff --git a/siteactive.py b/siteactive.py new file mode 100644 index 000000000..ca530bf49 --- /dev/null +++ b/siteactive.py @@ -0,0 +1,121 @@ +__filename__ = "siteactive.py" +__author__ = "Bob Mottram" +__credits__ = ["webchk"] +__license__ = "AGPL3+" +__version__ = "1.2.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import http.client +from urllib.parse import urlparse +import ssl + + +class Result: + """Holds result of an URL check. + + The redirect attribute is a Result object that the URL was redirected to. + + The sitemap_urls attribute will contain a list of Result object if url + is a sitemap file and http_response() was run with parse set to True. + """ + def __init__(self, url): + self.url = url + self.status = 0 + self.desc = '' + self.headers = None + self.latency = 0 + self.content = '' + self.redirect = None + self.sitemap_urls = None + + def __repr__(self): + if self.status == 0: + return '{} ... {}'.format(self.url, self.desc) + return '{} ... {} {} ({})'.format( + self.url, self.status, self.desc, self.latency + ) + + def fill_headers(self, headers): + """Takes a list of tuples and converts it a dictionary.""" + self.headers = {h[0]: h[1] for h in headers} + + +def _siteActiveParseUrl(url): + """Returns an object with properties representing + + scheme: URL scheme specifier + netloc: Network location part + path: Hierarchical path + params: Parameters for last path element + query: Query component + fragment: Fragment identifier + username: User name + password: Password + hostname: Host name (lower case) + port: Port number as integer, if present + """ + loc = urlparse(url) + + # if the scheme (http, https ...) is not available urlparse wont work + if loc.scheme == "": + url = "http://" + url + loc = urlparse(url) + return loc + + +def _siteACtiveHttpConnect(loc, timeout: int): + """Connects to the host and returns an HTTP or HTTPS connections.""" + if loc.scheme == "https": + ssl_context = ssl.SSLContext() + return http.client.HTTPSConnection( + loc.netloc, context=ssl_context, timeout=timeout) + return http.client.HTTPConnection(loc.netloc, timeout=timeout) + + +def _siteActiveHttpRequest(loc, timeout: int): + """Performs a HTTP request and return response in a Result object. + """ + conn = _siteACtiveHttpConnect(loc, timeout) + method = 'HEAD' + + conn.request(method, loc.path) + resp = conn.getresponse() + + result = Result(loc.geturl()) + result.status = resp.status + result.desc = resp.reason + result.fill_headers(resp.getheaders()) + + conn.close() + return result + + +def siteIsActive(url: str, timeout=10) -> bool: + """Returns true if the current url is resolvable. + This can be used to check that an instance is online before + trying to send posts to it. + """ + if not url.startswith('http'): + return False + if '.onion/' in url or '.i2p/' in url or \ + url.endswith('.onion') or \ + url.endswith('.i2p'): + # skip this check for onion and i2p + return True + + loc = _siteActiveParseUrl(url) + result = Result(url=url) + + try: + result = _siteActiveHttpRequest(loc, timeout) + + if 400 <= result.status < 500: + return result + + return True + + except BaseException: + pass + return False diff --git a/tests.py b/tests.py index 308e1fb0c..7186012a1 100644 --- a/tests.py +++ b/tests.py @@ -38,7 +38,7 @@ from utils import getFullDomain from utils import validNickname from utils import firstParagraphFromString from utils import removeIdEnding -from utils import siteIsActive +from siteactive import siteIsActive from utils import updateRecentPostsCache from utils import followPerson from utils import getNicknameFromActor @@ -2067,6 +2067,7 @@ def testJsonld(): def testSiteIsActive(): print('testSiteIsActive') + assert(siteIsActive('https://archive.org')) assert(siteIsActive('https://mastodon.social')) assert(not siteIsActive('https://notarealwebsite.a.b.c')) @@ -2818,7 +2819,8 @@ def testFunctions(): 'createServerBob', 'createServerEve', 'E2EEremoveDevice', - 'setOrganizationScheme' + 'setOrganizationScheme', + 'fill_headers' ] excludeImports = [ 'link', diff --git a/theme/default/icons/separator_right.png b/theme/default/icons/separator_right.png new file mode 100644 index 000000000..3f31f0fce Binary files /dev/null and b/theme/default/icons/separator_right.png differ diff --git a/theme/default/theme.json b/theme/default/theme.json index 4d3a1cc47..62c1e5817 100644 --- a/theme/default/theme.json +++ b/theme/default/theme.json @@ -1,4 +1,6 @@ { + "post-separator-margin-top": "10px", + "post-separator-margin-bottom": "10px", "newswire-publish-icon": "True", "full-width-timeline-buttons": "False", "icons-as-buttons": "False", diff --git a/utils.py b/utils.py index 0f3d811cf..5a58d45ce 100644 --- a/utils.py +++ b/utils.py @@ -11,9 +11,6 @@ import time import shutil import datetime import json -from socket import error as SocketError -import errno -import urllib.request import idna from pprint import pprint from calendar import monthrange @@ -21,6 +18,13 @@ from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes +# posts containing these strings will always get screened out, +# both incoming and outgoing. +# Could include dubious clacks or admin dogwhistles +invalidCharacters = ( + '卐', '卍', '࿕', '࿖', '࿗', '࿘' +) + def getSHA256(msg: str): """Returns a SHA256 hash of the given string @@ -517,17 +521,23 @@ def isEvil(domain: str) -> bool: def containsInvalidChars(jsonStr: str) -> bool: """Does the given json string contain invalid characters? - e.g. dubious clacks/admin dogwhistles """ - invalidStrings = { - '卐', '卍', '࿕', '࿖', '࿗', '࿘' - } - for isInvalid in invalidStrings: + for isInvalid in invalidCharacters: if isInvalid in jsonStr: return True return False +def removeInvalidChars(text: str) -> str: + """Removes any invalid characters from a string + """ + for isInvalid in invalidCharacters: + if isInvalid not in text: + continue + text = text.replace(isInvalid, '') + return text + + def createPersonDir(nickname: str, domain: str, baseDir: str, dirname: str) -> str: """Create a directory for a person @@ -1841,28 +1851,6 @@ def updateAnnounceCollection(recentPostsCache: {}, saveJson(postJsonObject, postFilename) -def siteIsActive(url: str) -> bool: - """Returns true if the current url is resolvable. - This can be used to check that an instance is online before - trying to send posts to it. - """ - if not url.startswith('http'): - return False - if '.onion/' in url or '.i2p/' in url or \ - url.endswith('.onion') or \ - url.endswith('.i2p'): - # skip this check for onion and i2p - return True - try: - req = urllib.request.Request(url) - urllib.request.urlopen(req, timeout=10) # nosec - return True - except SocketError as e: - if e.errno == errno.ECONNRESET: - print('WARN: connection was reset during siteIsActive') - return False - - def weekDayOfMonthStart(monthNumber: int, year: int) -> int: """Gets the day number of the first day of the month 1=sun, 7=sat diff --git a/webapp_hashtagswarm.py b/webapp_hashtagswarm.py index 19f08661f..f2ffb577d 100644 --- a/webapp_hashtagswarm.py +++ b/webapp_hashtagswarm.py @@ -203,8 +203,12 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str: categoryStr = \ getHashtagCategory(baseDir, hashTagName) if len(categoryStr) < maxTagLength: - if categoryStr not in categorySwarm: - categorySwarm.append(categoryStr) + if '#' not in categoryStr and \ + '&' not in categoryStr and \ + '"' not in categoryStr and \ + "'" not in categoryStr: + if categoryStr not in categorySwarm: + categorySwarm.append(categoryStr) break break diff --git a/webapp_post.py b/webapp_post.py index dd4c9a24b..c9505c819 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -327,9 +327,9 @@ def _getEditIconHtml(baseDir: str, nickname: str, domainFull: str, """ editStr = '' actor = postJsonObject['actor'] - if (actor.endswith(domainFull + '/users/' + nickname) or + if (actor.endswith('/' + domainFull + '/users/' + nickname) or (isEditor(baseDir, nickname) and - actor.endswith(domainFull + '/users/news'))): + actor.endswith('/' + domainFull + '/users/news'))): postId = postJsonObject['object']['id']