Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon into main

daemon.py

@@ -255,6 +255,7 @@ from newswire import rss2Footer
 from newswire import loadHashtagCategories
 from newsdaemon import runNewswireWatchdog
 from newsdaemon import runNewswireDaemon
+from newsdaemon import refreshNewswire
 from filters import isFiltered
 from filters import addGlobalFilter
 from filters import removeGlobalFilter
@@ -392,7 +393,7 @@ class PubServer(BaseHTTPRequestHandler):
                              schedulePost,
                              eventDate,
                              eventTime,
-                             location)
+                             location, False)
         if messageJson:
             # name field contains the answer
             messageJson['object']['name'] = answer
@@ -12373,7 +12374,7 @@ class PubServer(BaseHTTPRequestHandler):
                                      fields['replyTo'], fields['replyTo'],
                                      fields['subject'], fields['schedulePost'],
                                      fields['eventDate'], fields['eventTime'],
-                                     fields['location'])
+                                     fields['location'], False)
                 if messageJson:
                     if fields['schedulePost']:
                         return 1
@@ -12419,6 +12420,12 @@ class PubServer(BaseHTTPRequestHandler):
                         return 1
                     else:
                         return -1
+                if not fields['subject']:
+                    print('WARN: blog posts must have a title')
+                    return -1
+                if not fields['message']:
+                    print('WARN: blog posts must have content')
+                    return -1
                 # submit button on newblog screen
                 messageJson = \
                     createBlogPost(self.server.baseDir, nickname,
@@ -12438,6 +12445,7 @@ class PubServer(BaseHTTPRequestHandler):
                     if fields['schedulePost']:
                         return 1
                     if self._postToOutbox(messageJson, __version__, nickname):
+                        refreshNewswire(self.server.baseDir)
                         populateReplies(self.server.baseDir,
                                         self.server.httpPrefix,
                                         self.server.domainFull,

newsdaemon.py

@@ -660,6 +660,7 @@ def runNewswireDaemon(baseDir: str, httpd,
     """Periodically updates RSS feeds
     """
     newswireStateFilename = baseDir + '/accounts/.newswirestate.json'
+    refreshFilename = baseDir + '/accounts/.refresh_newswire'
 
     # initial sleep to allow the system to start up
     time.sleep(50)
@@ -722,7 +723,16 @@ def runNewswireDaemon(baseDir: str, httpd,
                                   httpd.maxNewsPosts)
 
         # wait a while before the next feeds update
-        time.sleep(1200)
+        for tick in range(120):
+            time.sleep(10)
+            # if a new blog post has been created then stop
+            # waiting and recalculate the newswire
+            if os.path.isfile(refreshFilename):
+                try:
+                    os.remove(refreshFilename)
+                except BaseException:
+                    pass
+                break
 
 
 def runNewswireWatchdog(projectVersion: str, httpd) -> None:
@@ -740,3 +750,15 @@ def runNewswireWatchdog(projectVersion: str, httpd) -> None:
                 newswireOriginal.clone(runNewswireDaemon)
             httpd.thrNewswireDaemon.start()
             print('Restarting newswire daemon...')
+
+
+def refreshNewswire(baseDir: str) -> None:
+    """Causes the newswire to be updated.
+    This creates a file which is then detected by the daemon
+    """
+    refreshFilename = baseDir + '/accounts/.refresh_newswire'
+    if os.path.isfile(refreshFilename):
+        return
+    refreshFile = open(refreshFilename, 'w+')
+    refreshFile.write('\n')
+    refreshFile.close()

posts.py

@@ -30,6 +30,8 @@ from session import postJsonString
 from session import postImage
 from webfinger import webfingerHandle
 from httpsig import createSignedHeader
+from siteactive import siteIsActive
+from utils import removeInvalidChars
 from utils import fileLastModified
 from utils import isPublicPost
 from utils import hasUsersPath
@@ -38,7 +40,6 @@ from utils import getFullDomain
 from utils import getFollowersList
 from utils import isEvil
 from utils import removeIdEnding
-from utils import siteIsActive
 from utils import getCachedPostFilename
 from utils import getStatusNumber
 from utils import createPersonDir
@@ -823,7 +824,7 @@ def validContentWarning(cw: str) -> str:
     # so remove them
     if '#' in cw:
         cw = cw.replace('#', '').replace('  ', ' ')
-    return cw
+    return removeInvalidChars(cw)
 
 
 def _loadAutoCW(baseDir: str, nickname: str, domain: str) -> []:
@@ -880,6 +881,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
                     eventStatus=None, ticketUrl=None) -> {}:
     """Creates a message
     """
+    content = removeInvalidChars(content)
+
     subject = _addAutoCW(baseDir, nickname, domain, subject, content)
 
     if nickname != 'news':
@@ -924,7 +927,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
     sensitive = False
     summary = None
     if subject:
-        summary = validContentWarning(subject)
+        summary = removeInvalidChars(validContentWarning(subject))
         sensitive = True
 
     toRecipients = []
@@ -1047,6 +1050,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
     postObjectType = 'Note'
     if eventUUID:
         postObjectType = 'Event'
+    if isArticle:
+        postObjectType = 'Article'
 
     if not clientToServer:
         actorUrl = httpPrefix + '://' + domain + '/users/' + nickname
@@ -1389,10 +1394,22 @@ def createPublicPost(baseDir: str,
                      imageDescription: str,
                      inReplyTo=None, inReplyToAtomUri=None, subject=None,
                      schedulePost=False,
-                     eventDate=None, eventTime=None, location=None) -> {}:
+                     eventDate=None, eventTime=None, location=None,
+                     isArticle=False) -> {}:
     """Public post
     """
     domainFull = getFullDomain(domain, port)
+    isModerationReport = False
+    eventUUID = None
+    category = None
+    joinMode = None
+    endDate = None
+    endTime = None
+    maximumAttendeeCapacity = None
+    repliesModerationOption = None
+    anonymousParticipationEnabled = None
+    eventStatus = None
+    ticketUrl = None
     return _createPostBase(baseDir, nickname, domain, port,
                            'https://www.w3.org/ns/activitystreams#Public',
                            httpPrefix + '://' + domainFull + '/users/' +
@@ -1401,38 +1418,27 @@ def createPublicPost(baseDir: str,
                            clientToServer, commentsEnabled,
                            attachImageFilename, mediaType,
                            imageDescription,
-                           False, False, inReplyTo, inReplyToAtomUri, subject,
+                           isModerationReport, isArticle,
-                           schedulePost, eventDate, eventTime, location,
-                           None, None, None, None, None,
-                           None, None, None, None, None)
-
-
-def createBlogPost(baseDir: str,
-                   nickname: str, domain: str, port: int, httpPrefix: str,
-                   content: str, followersOnly: bool, saveToFile: bool,
-                   clientToServer: bool, commentsEnabled: bool,
-                   attachImageFilename: str, mediaType: str,
-                   imageDescription: str,
-                   inReplyTo=None, inReplyToAtomUri=None, subject=None,
-                   schedulePost=False,
-                   eventDate=None, eventTime=None, location=None) -> {}:
-    blog = \
-        createPublicPost(baseDir,
-                         nickname, domain, port, httpPrefix,
-                         content, followersOnly, saveToFile,
-                         clientToServer, commentsEnabled,
-                         attachImageFilename, mediaType,
-                         imageDescription,
                            inReplyTo, inReplyToAtomUri, subject,
-                         schedulePost,
+                           schedulePost, eventDate, eventTime, location,
-                         eventDate, eventTime, location)
+                           eventUUID, category, joinMode, endDate, endTime,
-    blog['object']['type'] = 'Article'
+                           maximumAttendeeCapacity,
+                           repliesModerationOption,
+                           anonymousParticipationEnabled,
+                           eventStatus, ticketUrl)
 
+
+def _appendCitationsToBlogPost(baseDir: str,
+                               nickname: str, domain: str,
+                               blogJson: {}) -> None:
+    """Appends any citations to a new blog post
+    """
     # append citations tags, stored in a file
     citationsFilename = \
         baseDir + '/accounts/' + \
         nickname + '@' + domain + '/.citations.txt'
-    if os.path.isfile(citationsFilename):
+    if not os.path.isfile(citationsFilename):
+        return
     citationsSeparator = '#####'
     with open(citationsFilename, "r") as f:
         citations = f.readlines()
@@ -1450,9 +1456,32 @@ def createBlogPost(baseDir: str,
                 "name": title,
                 "url": link
             }
-                blog['object']['tag'].append(tagJson)
+            blogJson['object']['tag'].append(tagJson)
 
-    return blog
+
+def createBlogPost(baseDir: str,
+                   nickname: str, domain: str, port: int, httpPrefix: str,
+                   content: str, followersOnly: bool, saveToFile: bool,
+                   clientToServer: bool, commentsEnabled: bool,
+                   attachImageFilename: str, mediaType: str,
+                   imageDescription: str,
+                   inReplyTo=None, inReplyToAtomUri=None, subject=None,
+                   schedulePost=False,
+                   eventDate=None, eventTime=None, location=None) -> {}:
+    blogJson = \
+        createPublicPost(baseDir,
+                         nickname, domain, port, httpPrefix,
+                         content, followersOnly, saveToFile,
+                         clientToServer, commentsEnabled,
+                         attachImageFilename, mediaType,
+                         imageDescription,
+                         inReplyTo, inReplyToAtomUri, subject,
+                         schedulePost,
+                         eventDate, eventTime, location, True)
+
+    _appendCitationsToBlogPost(baseDir, nickname, domain, blogJson)
+
+    return blogJson
 
 
 def createNewsPost(baseDir: str,
@@ -1477,7 +1506,7 @@ def createNewsPost(baseDir: str,
                          imageDescription,
                          inReplyTo, inReplyToAtomUri, subject,
                          schedulePost,
-                         eventDate, eventTime, location)
+                         eventDate, eventTime, location, True)
     blog['object']['type'] = 'Article'
     return blog
 

siteactive.py

@@ -0,0 +1,121 @@
+__filename__ = "siteactive.py"
+__author__ = "Bob Mottram"
+__credits__ = ["webchk"]
+__license__ = "AGPL3+"
+__version__ = "1.2.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@freedombone.net"
+__status__ = "Production"
+
+import http.client
+from urllib.parse import urlparse
+import ssl
+
+
+class Result:
+    """Holds result of an URL check.
+
+    The redirect attribute is a Result object that the URL was redirected to.
+
+    The sitemap_urls attribute will contain a list of Result object if url
+    is a sitemap file and http_response() was run with parse set to True.
+    """
+    def __init__(self, url):
+        self.url = url
+        self.status = 0
+        self.desc = ''
+        self.headers = None
+        self.latency = 0
+        self.content = ''
+        self.redirect = None
+        self.sitemap_urls = None
+
+    def __repr__(self):
+        if self.status == 0:
+            return '{} ... {}'.format(self.url, self.desc)
+        return '{} ... {} {} ({})'.format(
+            self.url, self.status, self.desc, self.latency
+        )
+
+    def fill_headers(self, headers):
+        """Takes a list of tuples and converts it a dictionary."""
+        self.headers = {h[0]: h[1] for h in headers}
+
+
+def _siteActiveParseUrl(url):
+    """Returns an object with properties representing
+
+    scheme:   URL scheme specifier
+    netloc:   Network location part
+    path:     Hierarchical path
+    params:   Parameters for last path element
+    query:    Query component
+    fragment: Fragment identifier
+    username: User name
+    password: Password
+    hostname: Host name (lower case)
+    port:     Port number as integer, if present
+    """
+    loc = urlparse(url)
+
+    # if the scheme (http, https ...) is not available urlparse wont work
+    if loc.scheme == "":
+        url = "http://" + url
+        loc = urlparse(url)
+    return loc
+
+
+def _siteACtiveHttpConnect(loc, timeout: int):
+    """Connects to the host and returns an HTTP or HTTPS connections."""
+    if loc.scheme == "https":
+        ssl_context = ssl.SSLContext()
+        return http.client.HTTPSConnection(
+            loc.netloc, context=ssl_context, timeout=timeout)
+    return http.client.HTTPConnection(loc.netloc, timeout=timeout)
+
+
+def _siteActiveHttpRequest(loc, timeout: int):
+    """Performs a HTTP request and return response in a Result object.
+    """
+    conn = _siteACtiveHttpConnect(loc, timeout)
+    method = 'HEAD'
+
+    conn.request(method, loc.path)
+    resp = conn.getresponse()
+
+    result = Result(loc.geturl())
+    result.status = resp.status
+    result.desc = resp.reason
+    result.fill_headers(resp.getheaders())
+
+    conn.close()
+    return result
+
+
+def siteIsActive(url: str, timeout=10) -> bool:
+    """Returns true if the current url is resolvable.
+    This can be used to check that an instance is online before
+    trying to send posts to it.
+    """
+    if not url.startswith('http'):
+        return False
+    if '.onion/' in url or '.i2p/' in url or \
+       url.endswith('.onion') or \
+       url.endswith('.i2p'):
+        # skip this check for onion and i2p
+        return True
+
+    loc = _siteActiveParseUrl(url)
+    result = Result(url=url)
+
+    try:
+        result = _siteActiveHttpRequest(loc, timeout)
+
+        if 400 <= result.status < 500:
+            return result
+
+        return True
+
+    except BaseException:
+        pass
+    return False

tests.py

@@ -38,7 +38,7 @@ from utils import getFullDomain
 from utils import validNickname
 from utils import firstParagraphFromString
 from utils import removeIdEnding
-from utils import siteIsActive
+from siteactive import siteIsActive
 from utils import updateRecentPostsCache
 from utils import followPerson
 from utils import getNicknameFromActor
@@ -2067,6 +2067,7 @@ def testJsonld():
 
 def testSiteIsActive():
     print('testSiteIsActive')
+    assert(siteIsActive('https://archive.org'))
     assert(siteIsActive('https://mastodon.social'))
     assert(not siteIsActive('https://notarealwebsite.a.b.c'))
 
@@ -2818,7 +2819,8 @@ def testFunctions():
         'createServerBob',
         'createServerEve',
         'E2EEremoveDevice',
-        'setOrganizationScheme'
+        'setOrganizationScheme',
+        'fill_headers'
     ]
     excludeImports = [
         'link',

theme/default/theme.json

@@ -1,4 +1,6 @@
 {
+    "post-separator-margin-top": "10px",
+    "post-separator-margin-bottom": "10px",
     "newswire-publish-icon": "True",
     "full-width-timeline-buttons": "False",
     "icons-as-buttons": "False",

utils.py

@@ -11,9 +11,6 @@ import time
 import shutil
 import datetime
 import json
-from socket import error as SocketError
-import errno
-import urllib.request
 import idna
 from pprint import pprint
 from calendar import monthrange
@@ -21,6 +18,13 @@ from followingCalendar import addPersonToCalendar
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 
+# posts containing these strings will always get screened out,
+# both incoming and outgoing.
+# Could include dubious clacks or admin dogwhistles
+invalidCharacters = (
+    '卐', '卍', '࿕', '࿖', '࿗', '࿘'
+)
+
 
 def getSHA256(msg: str):
     """Returns a SHA256 hash of the given string
@@ -517,17 +521,23 @@ def isEvil(domain: str) -> bool:
 
 def containsInvalidChars(jsonStr: str) -> bool:
     """Does the given json string contain invalid characters?
-    e.g. dubious clacks/admin dogwhistles
     """
-    invalidStrings = {
+    for isInvalid in invalidCharacters:
-        '卐', '卍', '࿕', '࿖', '࿗', '࿘'
-    }
-    for isInvalid in invalidStrings:
         if isInvalid in jsonStr:
             return True
     return False
 
 
+def removeInvalidChars(text: str) -> str:
+    """Removes any invalid characters from a string
+    """
+    for isInvalid in invalidCharacters:
+        if isInvalid not in text:
+            continue
+        text = text.replace(isInvalid, '')
+    return text
+
+
 def createPersonDir(nickname: str, domain: str, baseDir: str,
                     dirname: str) -> str:
     """Create a directory for a person
@@ -1841,28 +1851,6 @@ def updateAnnounceCollection(recentPostsCache: {},
         saveJson(postJsonObject, postFilename)
 
 
-def siteIsActive(url: str) -> bool:
-    """Returns true if the current url is resolvable.
-    This can be used to check that an instance is online before
-    trying to send posts to it.
-    """
-    if not url.startswith('http'):
-        return False
-    if '.onion/' in url or '.i2p/' in url or \
-       url.endswith('.onion') or \
-       url.endswith('.i2p'):
-        # skip this check for onion and i2p
-        return True
-    try:
-        req = urllib.request.Request(url)
-        urllib.request.urlopen(req, timeout=10)  # nosec
-        return True
-    except SocketError as e:
-        if e.errno == errno.ECONNRESET:
-            print('WARN: connection was reset during siteIsActive')
-    return False
-
-
 def weekDayOfMonthStart(monthNumber: int, year: int) -> int:
     """Gets the day number of the first day of the month
     1=sun, 7=sat

webapp_hashtagswarm.py

@@ -203,6 +203,10 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str:
                             categoryStr = \
                                 getHashtagCategory(baseDir, hashTagName)
                             if len(categoryStr) < maxTagLength:
+                                if '#' not in categoryStr and \
+                                   '&' not in categoryStr and \
+                                   '"' not in categoryStr and \
+                                   "'" not in categoryStr:
                                     if categoryStr not in categorySwarm:
                                         categorySwarm.append(categoryStr)
                         break

webapp_post.py

@@ -327,9 +327,9 @@ def _getEditIconHtml(baseDir: str, nickname: str, domainFull: str,
     """
     editStr = ''
     actor = postJsonObject['actor']
-    if (actor.endswith(domainFull + '/users/' + nickname) or
+    if (actor.endswith('/' + domainFull + '/users/' + nickname) or
         (isEditor(baseDir, nickname) and
-         actor.endswith(domainFull + '/users/news'))):
+         actor.endswith('/' + domainFull + '/users/news'))):
 
         postId = postJsonObject['object']['id']