Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon

Makefile

@@ -23,4 +23,4 @@ clean:
 	rm -f deploy/*~
 	rm -f translations/*~
 	rm -rf __pycache__
-	rm calendar.css blog.css epicyon.css follow.css login.css options.css search.css suspended.css
+	rm -f calendar.css blog.css epicyon.css follow.css login.css options.css search.css suspended.css

blocking.py

@@ -7,6 +7,9 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import os
+from datetime import datetime
+from utils import fileLastModified
+from utils import setConfigParam
 from utils import hasUsersPath
 from utils import getFullDomain
 from utils import removeIdEnding
@@ -175,15 +178,27 @@ def isBlockedDomain(baseDir: str, domain: str) -> bool:
     if noOfSections > 2:
         shortDomain = domain[noOfSections-2] + '.' + domain[noOfSections-1]
 
-    globalBlockingFilename = baseDir + '/accounts/blocking.txt'
+    allowFilename = baseDir + '/accounts/allowedinstances.txt'
-    if os.path.isfile(globalBlockingFilename):
+    if not os.path.isfile(allowFilename):
-        with open(globalBlockingFilename, 'r') as fpBlocked:
+        # instance block list
-            blockedStr = fpBlocked.read()
+        globalBlockingFilename = baseDir + '/accounts/blocking.txt'
-            if '*@' + domain in blockedStr:
+        if os.path.isfile(globalBlockingFilename):
-                return True
+            with open(globalBlockingFilename, 'r') as fpBlocked:
-            if shortDomain:
+                blockedStr = fpBlocked.read()
-                if '*@' + shortDomain in blockedStr:
+                if '*@' + domain in blockedStr:
                     return True
+                if shortDomain:
+                    if '*@' + shortDomain in blockedStr:
+                        return True
+    else:
+        # instance allow list
+        if not shortDomain:
+            if domain not in open(allowFilename).read():
+                return True
+        else:
+            if shortDomain not in open(allowFilename).read():
+                return True
+
     return False
 
 
@@ -344,3 +359,91 @@ def outboxUndoBlock(baseDir: str, httpPrefix: str,
                 nicknameBlocked, domainBlockedFull)
     if debug:
         print('DEBUG: post undo blocked via c2s - ' + postFilename)
+
+
+def setBrochMode(baseDir: str, domainFull: str, enabled: bool) -> None:
+    """Broch mode can be used to lock down the instance during
+    a period of time when it is temporarily under attack.
+    For example, where an adversary is constantly spinning up new
+    instances.
+    It surveys the following lists of all accounts and uses that
+    to construct an instance level allow list. Anything arriving
+    which is then not from one of the allowed domains will be dropped
+    """
+    allowFilename = baseDir + '/accounts/allowedinstances.txt'
+
+    if not enabled:
+        # remove instance allow list
+        if os.path.isfile(allowFilename):
+            os.remove(allowFilename)
+            print('Broch mode turned off')
+    else:
+        if os.path.isfile(allowFilename):
+            lastModified = fileLastModified(allowFilename)
+            print('Broch mode already activated ' + lastModified)
+            return
+        # generate instance allow list
+        allowedDomains = [domainFull]
+        followFiles = ('following.txt', 'followers.txt')
+        for subdir, dirs, files in os.walk(baseDir + '/accounts'):
+            for acct in dirs:
+                if '@' not in acct:
+                    continue
+                if 'inbox@' in acct or 'news@' in acct:
+                    continue
+                accountDir = os.path.join(baseDir + '/accounts', acct)
+                for followFileType in followFiles:
+                    followingFilename = accountDir + '/' + followFileType
+                    if not os.path.isfile(followingFilename):
+                        continue
+                    with open(followingFilename, "r") as f:
+                        followList = f.readlines()
+                        for handle in followList:
+                            if '@' not in handle:
+                                continue
+                            handle = handle.replace('\n', '')
+                            handleDomain = handle.split('@')[1]
+                            if handleDomain not in allowedDomains:
+                                allowedDomains.append(handleDomain)
+            break
+
+        # write the allow file
+        allowFile = open(allowFilename, "w+")
+        if allowFile:
+            allowFile.write(domainFull + '\n')
+            for d in allowedDomains:
+                allowFile.write(d + '\n')
+            allowFile.close()
+            print('Broch mode enabled')
+
+    setConfigParam(baseDir, "brochMode", enabled)
+
+
+def brochModeLapses(baseDir: str, lapseDays=7) -> bool:
+    """After broch mode is enabled it automatically
+    elapses after a period of time
+    """
+    allowFilename = baseDir + '/accounts/allowedinstances.txt'
+    if not os.path.isfile(allowFilename):
+        return False
+    lastModified = fileLastModified(allowFilename)
+    modifiedDate = None
+    brochMode = True
+    try:
+        modifiedDate = \
+            datetime.strptime(lastModified, "%Y-%m-%dT%H:%M:%SZ")
+    except BaseException:
+        return brochMode
+    if not modifiedDate:
+        return brochMode
+    currTime = datetime.datetime.utcnow()
+    daysSinceBroch = (currTime - modifiedDate).days
+    if daysSinceBroch >= lapseDays:
+        try:
+            os.remove(allowFilename)
+            brochMode = False
+            setConfigParam(baseDir, "brochMode", brochMode)
+            print('Broch mode has elapsed')
+        except BaseException:
+            pass
+    return brochMode

cache.py

@@ -8,11 +8,45 @@ __status__ = "Production"
 
 import os
 import datetime
+from session import urlExists
 from utils import loadJson
 from utils import saveJson
 from utils import getFileCaseInsensitive
 
 
+def _removePersonFromCache(baseDir: str, personUrl: str,
+                           personCache: {}) -> bool:
+    """Removes an actor from the cache
+    """
+    cacheFilename = baseDir + '/cache/actors/' + \
+        personUrl.replace('/', '#')+'.json'
+    if os.path.isfile(cacheFilename):
+        try:
+            os.remove(cacheFilename)
+        except BaseException:
+            pass
+    if personCache.get(personUrl):
+        del personCache[personUrl]
+
+
+def checkForChangedActor(session, baseDir: str,
+                         httpPrefix: str, domainFull: str,
+                         personUrl: str, avatarUrl: str, personCache: {},
+                         timeoutSec: int):
+    """Checks if the avatar url exists and if not then
+    the actor has probably changed without receiving an actor/Person Update.
+    So clear the actor from the cache and it will be refreshed when the next
+    post from them is sent
+    """
+    if not session or not avatarUrl:
+        return
+    if domainFull in avatarUrl:
+        return
+    if urlExists(session, avatarUrl, timeoutSec, httpPrefix, domainFull):
+        return
+    _removePersonFromCache(baseDir, personUrl, personCache)
+
+
 def storePersonInCache(baseDir: str, personUrl: str,
                        personJson: {}, personCache: {},
                        allowWriteToFile: bool) -> None:

content.py

@@ -788,6 +788,13 @@ def addHtmlTags(baseDir: str, httpPrefix: str,
                     prevWordStr = ''
                     continue
             elif firstChar == '#':
+                # remove any endings from the hashtag
+                hashTagEndings = ('.', ':', ';', '-', '\n')
+                for ending in hashTagEndings:
+                    if wordStr.endswith(ending):
+                        wordStr = wordStr[:len(wordStr) - 1]
+                        break
+
                 if _addHashTags(wordStr, httpPrefix, originalDomain,
                                 replaceHashTags, hashtags):
                     prevWordStr = ''

daemon.py

@@ -109,6 +109,7 @@ from threads import threadWithTrace
 from threads import removeDormantThreads
 from media import replaceYouTube
 from media import attachMedia
+from blocking import setBrochMode
 from blocking import addBlock
 from blocking import removeBlock
 from blocking import addGlobalBlock
@@ -185,6 +186,7 @@ from shares import addShare
 from shares import removeShare
 from shares import expireShares
 from categories import setHashtagCategory
+from utils import getLocalNetworkAddresses
 from utils import decodedHost
 from utils import isPublicPost
 from utils import getLockedAccount
@@ -218,6 +220,7 @@ from utils import loadJson
 from utils import saveJson
 from utils import isSuspended
 from utils import dangerousMarkup
+from utils import refreshNewswire
 from manualapprove import manualDenyFollowRequest
 from manualapprove import manualApproveFollowRequest
 from announce import createAnnounce
@@ -227,6 +230,7 @@ from content import extractMediaInFormPOST
 from content import saveMediaInFormPOST
 from content import extractTextFieldsInPOST
 from media import removeMetaData
+from cache import checkForChangedActor
 from cache import storePersonInCache
 from cache import getPersonFromCache
 from httpsig import verifyPostHeaders
@@ -392,7 +396,7 @@ class PubServer(BaseHTTPRequestHandler):
                              schedulePost,
                              eventDate,
                              eventTime,
-                             location)
+                             location, False)
         if messageJson:
             # name field contains the answer
             messageJson['object']['name'] = answer
@@ -476,6 +480,10 @@ class PubServer(BaseHTTPRequestHandler):
             if 'text/html' not in self.headers['Accept']:
                 return False
         if self.headers['Accept'].startswith('*'):
+            if self.headers.get('User-Agent'):
+                if 'ELinks' in self.headers['User-Agent'] or \
+                   'Lynx' in self.headers['User-Agent']:
+                    return True
             return False
         if 'json' in self.headers['Accept']:
             return False
@@ -1151,20 +1159,46 @@ class PubServer(BaseHTTPRequestHandler):
 
         # check for blocked domains so that they can be rejected early
         messageDomain = None
-        if messageJson.get('actor'):
+        if not messageJson.get('actor'):
-            messageDomain, messagePort = \
-                getDomainFromActor(messageJson['actor'])
-            if isBlockedDomain(self.server.baseDir, messageDomain):
-                print('POST from blocked domain ' + messageDomain)
-                self._400()
-                self.server.POSTbusy = False
-                return 3
-        else:
             print('Message arriving at inbox queue has no actor')
             self._400()
             self.server.POSTbusy = False
             return 3
 
+        # actor should be a string
+        if not isinstance(messageJson['actor'], str):
+            self._400()
+            self.server.POSTbusy = False
+            return 3
+
+        # actor should look like a url
+        if '://' not in messageJson['actor'] or \
+           '.' not in messageJson['actor']:
+            print('POST actor does not look like a url ' +
+                  messageJson['actor'])
+            self._400()
+            self.server.POSTbusy = False
+            return 3
+
+        # sent by an actor on a local network address?
+        if not self.server.allowLocalNetworkAccess:
+            localNetworkPatternList = getLocalNetworkAddresses()
+            for localNetworkPattern in localNetworkPatternList:
+                if localNetworkPattern in messageJson['actor']:
+                    print('POST actor contains local network address ' +
+                          messageJson['actor'])
+                    self._400()
+                    self.server.POSTbusy = False
+                    return 3
+
+        messageDomain, messagePort = \
+            getDomainFromActor(messageJson['actor'])
+        if isBlockedDomain(self.server.baseDir, messageDomain):
+            print('POST from blocked domain ' + messageDomain)
+            self._400()
+            self.server.POSTbusy = False
+            return 3
+
         # if the inbox queue is full then return a busy code
         if len(self.server.inboxQueue) >= self.server.maxQueueLength:
             if messageDomain:
@@ -1947,12 +1981,50 @@ class PubServer(BaseHTTPRequestHandler):
                 if postsToNews == 'on':
                     if os.path.isfile(newswireBlockedFilename):
                         os.remove(newswireBlockedFilename)
+                        refreshNewswire(self.server.baseDir)
                 else:
                     if os.path.isdir(accountDir):
                         noNewswireFile = open(newswireBlockedFilename, "w+")
                         if noNewswireFile:
                             noNewswireFile.write('\n')
                             noNewswireFile.close()
+                            refreshNewswire(self.server.baseDir)
+            usersPathStr = \
+                usersPath + '/' + self.server.defaultTimeline + \
+                '?page=' + str(pageNumber)
+            self._redirect_headers(usersPathStr, cookie,
+                                   callingDomain)
+            self.server.POSTbusy = False
+            return
+
+        # person options screen, permission to post to featured articles
+        # See htmlPersonOptions
+        if '&submitPostToFeatures=' in optionsConfirmParams:
+            adminNickname = getConfigParam(self.server.baseDir, 'admin')
+            if (chooserNickname != optionsNickname and
+                (chooserNickname == adminNickname or
+                 (isModerator(self.server.baseDir, chooserNickname) and
+                  not isModerator(self.server.baseDir, optionsNickname)))):
+                postsToFeatures = None
+                if 'postsToFeatures=' in optionsConfirmParams:
+                    postsToFeatures = \
+                        optionsConfirmParams.split('postsToFeatures=')[1]
+                    if '&' in postsToFeatures:
+                        postsToFeatures = postsToFeatures.split('&')[0]
+                accountDir = self.server.baseDir + '/accounts/' + \
+                    optionsNickname + '@' + optionsDomain
+                featuresBlockedFilename = accountDir + '/.nofeatures'
+                if postsToFeatures == 'on':
+                    if os.path.isfile(featuresBlockedFilename):
+                        os.remove(featuresBlockedFilename)
+                        refreshNewswire(self.server.baseDir)
+                else:
+                    if os.path.isdir(accountDir):
+                        noFeaturesFile = open(featuresBlockedFilename, "w+")
+                        if noFeaturesFile:
+                            noFeaturesFile.write('\n')
+                            noFeaturesFile.close()
+                            refreshNewswire(self.server.baseDir)
             usersPathStr = \
                 usersPath + '/' + self.server.defaultTimeline + \
                 '?page=' + str(pageNumber)
@@ -4472,6 +4544,18 @@ class PubServer(BaseHTTPRequestHandler):
                             setConfigParam(baseDir, "verifyAllSignatures",
                                            verifyAllSignatures)
 
+                            brochMode = False
+                            if fields.get('brochMode'):
+                                if fields['brochMode'] == 'on':
+                                    brochMode = True
+                            currBrochMode = \
+                                getConfigParam(baseDir, "brochMode")
+                            if brochMode != currBrochMode:
+                                setBrochMode(self.server.baseDir,
+                                             self.server.domainFull,
+                                             brochMode)
+                                setConfigParam(baseDir, "brochMode", brochMode)
+
                         # change moderators list
                         if fields.get('moderators'):
                             if path.startswith('/users/' +
@@ -5449,6 +5533,15 @@ class PubServer(BaseHTTPRequestHandler):
                 PGPfingerprint = getPGPfingerprint(actorJson)
                 if actorJson.get('alsoKnownAs'):
                     alsoKnownAs = actorJson['alsoKnownAs']
+
+            if self.server.session:
+                checkForChangedActor(self.server.session,
+                                     self.server.baseDir,
+                                     self.server.httpPrefix,
+                                     self.server.domainFull,
+                                     optionsActor, optionsProfileUrl,
+                                     self.server.personCache, 5)
+
             msg = htmlPersonOptions(self.server.defaultTimeline,
                                     self.server.cssCache,
                                     self.server.translate,
@@ -5468,7 +5561,9 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.dormantMonths,
                                     backToPath,
                                     lockedAccount,
-                                    movedTo, alsoKnownAs).encode('utf-8')
+                                    movedTo, alsoKnownAs,
+                                    self.server.textModeBanner,
+                                    self.server.newsInstance).encode('utf-8')
             msglen = len(msg)
             self._set_headers('text/html', msglen,
                               cookie, callingDomain)
@@ -10010,6 +10105,16 @@ class PubServer(BaseHTTPRequestHandler):
         # replace https://domain/@nick with https://domain/users/nick
         if self.path.startswith('/@'):
             self.path = self.path.replace('/@', '/users/')
+            # replace https://domain/@nick/statusnumber
+            # with https://domain/users/nick/statuses/statusnumber
+            nickname = self.path.split('/users/')[1]
+            if '/' in nickname:
+                statusNumberStr = nickname.split('/')[1]
+                if statusNumberStr.isdigit():
+                    nickname = nickname.split('/')[0]
+                    self.path = \
+                        self.path.replace('/users/' + nickname + '/',
+                                          '/users/' + nickname + '/statuses/')
 
         # turn off dropdowns on new post screen
         noDropDown = False
@@ -10038,9 +10143,13 @@ class PubServer(BaseHTTPRequestHandler):
 
         # manifest for progressive web apps
         if '/manifest.json' in self.path:
-            self._progressiveWebAppManifest(callingDomain,
+            if self._hasAccept(callingDomain):
-                                            GETstartTime, GETtimings)
+                if not self._requestHTTP():
-            return
+                    self._progressiveWebAppManifest(callingDomain,
+                                                    GETstartTime, GETtimings)
+                    return
+                else:
+                    self.path = '/'
 
         # default newswire favicon, for links to sites which
         # have no favicon
@@ -11084,7 +11193,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.translate,
                                    self.server.baseDir, self.path,
                                    self.server.httpPrefix,
-                                   self.server.domainFull).encode('utf-8')
+                                   self.server.domainFull,
+                                   self.server.textModeBanner).encode('utf-8')
                 msglen = len(msg)
                 self._set_headers('text/html', msglen, cookie, callingDomain)
                 self._write(msg)
@@ -12373,7 +12483,7 @@ class PubServer(BaseHTTPRequestHandler):
                                      fields['replyTo'], fields['replyTo'],
                                      fields['subject'], fields['schedulePost'],
                                      fields['eventDate'], fields['eventTime'],
-                                     fields['location'])
+                                     fields['location'], False)
                 if messageJson:
                     if fields['schedulePost']:
                         return 1
@@ -12419,6 +12529,12 @@ class PubServer(BaseHTTPRequestHandler):
                         return 1
                     else:
                         return -1
+                if not fields['subject']:
+                    print('WARN: blog posts must have a title')
+                    return -1
+                if not fields['message']:
+                    print('WARN: blog posts must have content')
+                    return -1
                 # submit button on newblog screen
                 messageJson = \
                     createBlogPost(self.server.baseDir, nickname,
@@ -12438,6 +12554,7 @@ class PubServer(BaseHTTPRequestHandler):
                     if fields['schedulePost']:
                         return 1
                     if self._postToOutbox(messageJson, __version__, nickname):
+                        refreshNewswire(self.server.baseDir)
                         populateReplies(self.server.baseDir,
                                         self.server.httpPrefix,
                                         self.server.domainFull,
@@ -13820,7 +13937,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None:
         break
 
 
-def runDaemon(verifyAllSignatures: bool,
+def runDaemon(brochMode: bool,
+              verifyAllSignatures: bool,
               sendThreadsTimeoutMins: int,
               dormantMonths: int,
               maxNewswirePosts: int,
@@ -14073,6 +14191,9 @@ def runDaemon(verifyAllSignatures: bool,
     # cache to store css files
     httpd.cssCache = {}
 
+    # whether to enable broch mode, which locks down the instance
+    setBrochMode(baseDir, httpd.domainFull, brochMode)
+
     if not os.path.isdir(baseDir + '/accounts/inbox@' + domain):
         print('Creating shared inbox: inbox@' + domain)
         createSharedInbox(baseDir, 'inbox', domain, port, httpPrefix)

emoji/default_emoji.json

@@ -1,4 +1,5 @@
 {
+    "android": "android",
     "popcorn": "1F37F",
     "1stplacemedal": "1F947",
     "abbutton": "1F18E",

epicyon-calendar.css

@@ -84,6 +84,14 @@ a:focus {
     border: 2px solid var(--focus-color);
 }
 
+.transparent {
+    color: transparent;
+    background: transparent;
+    font-size: 0px;
+    line-height: 0px;
+    height: 0px;
+}
+
 .calendar__day__header,
 .calendar__day__cell {
     border: 2px solid var(--lines-color);

epicyon-options.css

@@ -98,6 +98,14 @@ a:focus {
     border: 2px solid var(--focus-color);
 }
 
+.transparent {
+    color: transparent;
+    background: transparent;
+    font-size: 0px;
+    line-height: 0px;
+    height: 0px;
+}
+
 .follow {
     height: 100%;
     position: relative;

epicyon-profile.css

@@ -87,7 +87,7 @@
     --quote-right-margin: 0.1em;
     --quote-font-weight: normal;
     --quote-font-size: 120%;
-    --line-spacing: 130%;
+    --line-spacing: 180%;
     --line-spacing-newswire: 120%;
     --newswire-item-moderated-color: white;
     --newswire-date-moderated-color: white;
@@ -106,11 +106,11 @@
     --column-left-icons-margin: 0;
     --column-right-border-width: 0;
     --column-left-border-color: black;
-    --column-left-icon-size: 20%;
+    --column-left-icon-size: 2.1vw;
     --column-left-icon-size-mobile: 10%;
     --column-left-image-width-mobile: 40vw;
     --column-right-image-width-mobile: 100vw;
-    --column-right-icon-size: 20%;
+    --column-right-icon-size: 2.1vw;
     --column-right-icon-size-mobile: 10%;
     --newswire-date-color: white;
     --newswire-voted-background-color: black;

epicyon.py

@@ -279,6 +279,11 @@ parser.add_argument("--verifyAllSignatures",
                     const=True, default=False,
                     help="Whether to require that all incoming " +
                     "posts have valid jsonld signatures")
+parser.add_argument("--brochMode",
+                    dest='brochMode',
+                    type=str2bool, nargs='?',
+                    const=True, default=False,
+                    help="Enable broch mode")
 parser.add_argument("--noapproval", type=str2bool, nargs='?',
                     const=True, default=False,
                     help="Allow followers without approval")
@@ -2292,6 +2297,11 @@ verifyAllSignatures = \
 if verifyAllSignatures is not None:
     args.verifyAllSignatures = bool(verifyAllSignatures)
 
+brochMode = \
+    getConfigParam(baseDir, 'brochMode')
+if brochMode is not None:
+    args.brochMode = bool(brochMode)
+
 YTDomain = getConfigParam(baseDir, 'youtubedomain')
 if YTDomain:
     if '://' in YTDomain:
@@ -2305,7 +2315,8 @@ if setTheme(baseDir, themeName, domain, args.allowLocalNetworkAccess):
     print('Theme set to ' + themeName)
 
 if __name__ == "__main__":
-    runDaemon(args.verifyAllSignatures,
+    runDaemon(args.brochMode,
+              args.verifyAllSignatures,
               args.sendThreadsTimeoutMins,
               args.dormantMonths,
               args.maxNewswirePosts,

inbox.py

@@ -51,6 +51,7 @@ from bookmarks import updateBookmarksCollection
 from bookmarks import undoBookmarksCollectionEntry
 from blocking import isBlocked
 from blocking import isBlockedDomain
+from blocking import brochModeLapses
 from filters import isFiltered
 from utils import updateAnnounceCollection
 from utils import undoAnnounceCollectionEntry
@@ -2518,6 +2519,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
         # heartbeat to monitor whether the inbox queue is running
         heartBeatCtr += 5
         if heartBeatCtr >= 10:
+            # turn off broch mode after it has timed out
+            brochModeLapses(baseDir)
             print('>>> Heartbeat Q:' + str(len(queue)) + ' ' +
                   '{:%F %T}'.format(datetime.datetime.now()))
             heartBeatCtr = 0
@@ -2726,6 +2729,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
             print('DEBUG: checking http header signature')
             pprint(queueJson['httpHeaders'])
         postStr = json.dumps(queueJson['post'])
+        httpSignatureFailed = False
         if not verifyPostHeaders(httpPrefix,
                                  pubKey,
                                  queueJson['httpHeaders'],
@@ -2733,19 +2737,17 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
                                  queueJson['digest'],
                                  postStr,
                                  debug):
+            httpSignatureFailed = True
             print('Queue: Header signature check failed')
-            pprint(queueJson['httpHeaders'])
+            if debug:
-            if os.path.isfile(queueFilename):
+                pprint(queueJson['httpHeaders'])
-                os.remove(queueFilename)
+        else:
-            if len(queue) > 0:
+            if debug:
-                queue.pop(0)
+                print('DEBUG: http header signature check success')
-            continue
-
-        if debug:
-            print('DEBUG: http header signature check success')
 
         # check if a json signature exists on this post
-        checkJsonSignature = False
+        hasJsonSignature = False
+        jwebsigType = None
         originalJson = queueJson['original']
         if originalJson.get('@context') and \
            originalJson.get('signature'):
@@ -2754,41 +2756,59 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
                 jwebsig = originalJson['signature']
                 # signature exists and is of the expected type
                 if jwebsig.get('type') and jwebsig.get('signatureValue'):
-                    if jwebsig['type'] == 'RsaSignature2017':
+                    jwebsigType = jwebsig['type']
+                    if jwebsigType == 'RsaSignature2017':
                         if hasValidContext(originalJson):
-                            checkJsonSignature = True
+                            hasJsonSignature = True
                         else:
                             print('unrecognised @context: ' +
                                   str(originalJson['@context']))
 
         # strict enforcement of json signatures
-        if verifyAllSignatures and \
+        if not hasJsonSignature:
-           not checkJsonSignature:
+            if httpSignatureFailed:
-            print('inbox post does not have a jsonld signature ' +
+                if jwebsigType:
-                  keyId + ' ' + str(originalJson))
+                    print('Queue: Header signature check failed and does ' +
-            if os.path.isfile(queueFilename):
+                          'not have a recognised jsonld signature type ' +
-                os.remove(queueFilename)
+                          jwebsigType)
-            if len(queue) > 0:
-                queue.pop(0)
-            continue
-
-        if checkJsonSignature and verifyAllSignatures:
-            # use the original json message received, not one which may have
-            # been modified along the way
-            if not verifyJsonSignature(originalJson, pubKey):
-                if debug:
-                    print('WARN: jsonld inbox signature check failed ' +
-                          keyId + ' ' + pubKey + ' ' + str(originalJson))
                 else:
-                    print('WARN: jsonld inbox signature check failed ' +
+                    print('Queue: Header signature check failed and ' +
-                          keyId)
+                          'does not have jsonld signature')
+                if debug:
+                    pprint(queueJson['httpHeaders'])
+
+            if verifyAllSignatures:
+                print('Queue: inbox post does not have a jsonld signature ' +
+                      keyId + ' ' + str(originalJson))
+
+            if httpSignatureFailed or verifyAllSignatures:
                 if os.path.isfile(queueFilename):
                     os.remove(queueFilename)
                 if len(queue) > 0:
                     queue.pop(0)
                 continue
-            else:
+        else:
-                print('jsonld inbox signature check success ' + keyId)
+            if httpSignatureFailed or verifyAllSignatures:
+                # use the original json message received, not one which
+                # may have been modified along the way
+                if not verifyJsonSignature(originalJson, pubKey):
+                    if debug:
+                        print('WARN: jsonld inbox signature check failed ' +
+                              keyId + ' ' + pubKey + ' ' + str(originalJson))
+                    else:
+                        print('WARN: jsonld inbox signature check failed ' +
+                              keyId)
+                    if os.path.isfile(queueFilename):
+                        os.remove(queueFilename)
+                    if len(queue) > 0:
+                        queue.pop(0)
+                    continue
+                else:
+                    if httpSignatureFailed:
+                        print('jsonld inbox signature check success ' +
+                              'via relay ' + keyId)
+                    else:
+                        print('jsonld inbox signature check success ' + keyId)
 
         # set the id to the same as the post filename
         # This makes the filename and the id consistent

newsdaemon.py

@@ -660,6 +660,7 @@ def runNewswireDaemon(baseDir: str, httpd,
     """Periodically updates RSS feeds
     """
     newswireStateFilename = baseDir + '/accounts/.newswirestate.json'
+    refreshFilename = baseDir + '/accounts/.refresh_newswire'
 
     # initial sleep to allow the system to start up
     time.sleep(50)
@@ -722,7 +723,16 @@ def runNewswireDaemon(baseDir: str, httpd,
                                   httpd.maxNewsPosts)
 
         # wait a while before the next feeds update
-        time.sleep(1200)
+        for tick in range(120):
+            time.sleep(10)
+            # if a new blog post has been created then stop
+            # waiting and recalculate the newswire
+            if os.path.isfile(refreshFilename):
+                try:
+                    os.remove(refreshFilename)
+                except BaseException:
+                    pass
+                break
 
 
 def runNewswireWatchdog(projectVersion: str, httpd) -> None:

newswire.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import os
+import json
 import requests
 from socket import error as SocketError
 import errno
@@ -301,6 +302,7 @@ def _xml2StrToDict(baseDir: str, domain: str, xmlStr: str,
             continue
         title = rssItem.split('<title>')[1]
         title = _removeCDATA(title.split('</title>')[0])
+        title = removeHtml(title)
         description = ''
         if '<description>' in rssItem and '</description>' in rssItem:
             description = rssItem.split('<description>')[1]
@@ -332,12 +334,14 @@ def _xml2StrToDict(baseDir: str, domain: str, xmlStr: str,
                                       result, pubDateStr,
                                       title, link,
                                       votesStatus, postFilename,
-                                      description, moderated, mirrored)
+                                      description, moderated,
+                                      mirrored)
                 postCtr += 1
                 if postCtr >= maxPostsPerSource:
                     break
     if postCtr > 0:
-        print('Added ' + str(postCtr) + ' rss 2.0 feed items to newswire')
+        print('Added ' + str(postCtr) +
+              ' rss 2.0 feed items to newswire')
     return result
 
 
@@ -385,6 +389,7 @@ def _xml1StrToDict(baseDir: str, domain: str, xmlStr: str,
             continue
         title = rssItem.split('<title>')[1]
         title = _removeCDATA(title.split('</title>')[0])
+        title = removeHtml(title)
         description = ''
         if '<description>' in rssItem and '</description>' in rssItem:
             description = rssItem.split('<description>')[1]
@@ -416,12 +421,14 @@ def _xml1StrToDict(baseDir: str, domain: str, xmlStr: str,
                                       result, pubDateStr,
                                       title, link,
                                       votesStatus, postFilename,
-                                      description, moderated, mirrored)
+                                      description, moderated,
+                                      mirrored)
                 postCtr += 1
                 if postCtr >= maxPostsPerSource:
                     break
     if postCtr > 0:
-        print('Added ' + str(postCtr) + ' rss 1.0 feed items to newswire')
+        print('Added ' + str(postCtr) +
+              ' rss 1.0 feed items to newswire')
     return result
 
 
@@ -457,6 +464,7 @@ def _atomFeedToDict(baseDir: str, domain: str, xmlStr: str,
             continue
         title = atomItem.split('<title>')[1]
         title = _removeCDATA(title.split('</title>')[0])
+        title = removeHtml(title)
         description = ''
         if '<summary>' in atomItem and '</summary>' in atomItem:
             description = atomItem.split('<summary>')[1]
@@ -488,12 +496,124 @@ def _atomFeedToDict(baseDir: str, domain: str, xmlStr: str,
                                       result, pubDateStr,
                                       title, link,
                                       votesStatus, postFilename,
-                                      description, moderated, mirrored)
+                                      description, moderated,
+                                      mirrored)
                 postCtr += 1
                 if postCtr >= maxPostsPerSource:
                     break
     if postCtr > 0:
-        print('Added ' + str(postCtr) + ' atom feed items to newswire')
+        print('Added ' + str(postCtr) +
+              ' atom feed items to newswire')
+    return result
+
+
+def _jsonFeedV1ToDict(baseDir: str, domain: str, xmlStr: str,
+                      moderated: bool, mirrored: bool,
+                      maxPostsPerSource: int,
+                      maxFeedItemSizeKb: int) -> {}:
+    """Converts a json feed string to a dictionary
+    See https://jsonfeed.org/version/1.1
+    """
+    if '"items"' not in xmlStr:
+        return {}
+    try:
+        feedJson = json.loads(xmlStr)
+    except BaseException:
+        return {}
+    maxBytes = maxFeedItemSizeKb * 1024
+    if not feedJson.get('version'):
+        return {}
+    if not feedJson['version'].startswith('https://jsonfeed.org/version/1'):
+        return {}
+    if not feedJson.get('items'):
+        return {}
+    if not isinstance(feedJson['items'], list):
+        return {}
+    postCtr = 0
+    result = {}
+    for jsonFeedItem in feedJson['items']:
+        if not jsonFeedItem:
+            continue
+        if not isinstance(jsonFeedItem, dict):
+            continue
+        if not jsonFeedItem.get('url'):
+            continue
+        if not isinstance(jsonFeedItem['url'], str):
+            continue
+        if not jsonFeedItem.get('date_published'):
+            if not jsonFeedItem.get('date_modified'):
+                continue
+        if not jsonFeedItem.get('content_text'):
+            if not jsonFeedItem.get('content_html'):
+                continue
+        if jsonFeedItem.get('content_html'):
+            if not isinstance(jsonFeedItem['content_html'], str):
+                continue
+            title = removeHtml(jsonFeedItem['content_html'])
+        else:
+            if not isinstance(jsonFeedItem['content_text'], str):
+                continue
+            title = removeHtml(jsonFeedItem['content_text'])
+        if len(title) > maxBytes:
+            print('WARN: json feed title is too long')
+            continue
+        description = ''
+        if jsonFeedItem.get('description'):
+            if not isinstance(jsonFeedItem['description'], str):
+                continue
+            description = removeHtml(jsonFeedItem['description'])
+            if len(description) > maxBytes:
+                print('WARN: json feed description is too long')
+                continue
+            if jsonFeedItem.get('tags'):
+                if isinstance(jsonFeedItem['tags'], list):
+                    for tagName in jsonFeedItem['tags']:
+                        if not isinstance(tagName, str):
+                            continue
+                        if ' ' in tagName:
+                            continue
+                        if not tagName.startswith('#'):
+                            tagName = '#' + tagName
+                        if tagName not in description:
+                            description += ' ' + tagName
+
+        link = jsonFeedItem['url']
+        if '://' not in link:
+            continue
+        if len(link) > maxBytes:
+            print('WARN: json feed link is too long')
+            continue
+        itemDomain = link.split('://')[1]
+        if '/' in itemDomain:
+            itemDomain = itemDomain.split('/')[0]
+        if isBlockedDomain(baseDir, itemDomain):
+            continue
+        if jsonFeedItem.get('date_published'):
+            if not isinstance(jsonFeedItem['date_published'], str):
+                continue
+            pubDate = jsonFeedItem['date_published']
+        else:
+            if not isinstance(jsonFeedItem['date_modified'], str):
+                continue
+            pubDate = jsonFeedItem['date_modified']
+
+        pubDateStr = parseFeedDate(pubDate)
+        if pubDateStr:
+            if _validFeedDate(pubDateStr):
+                postFilename = ''
+                votesStatus = []
+                _addNewswireDictEntry(baseDir, domain,
+                                      result, pubDateStr,
+                                      title, link,
+                                      votesStatus, postFilename,
+                                      description, moderated,
+                                      mirrored)
+                postCtr += 1
+                if postCtr >= maxPostsPerSource:
+                    break
+    if postCtr > 0:
+        print('Added ' + str(postCtr) +
+              ' json feed items to newswire')
     return result
 
 
@@ -593,6 +713,10 @@ def _xmlStrToDict(baseDir: str, domain: str, xmlStr: str,
         return _atomFeedToDict(baseDir, domain,
                                xmlStr, moderated, mirrored,
                                maxPostsPerSource, maxFeedItemSizeKb)
+    elif 'https://jsonfeed.org/version/1' in xmlStr:
+        return _jsonFeedV1ToDict(baseDir, domain,
+                                 xmlStr, moderated, mirrored,
+                                 maxPostsPerSource, maxFeedItemSizeKb)
     return {}
 
 
@@ -794,7 +918,7 @@ def _addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str,
                     locatePost(baseDir, nickname,
                                domain, postUrl, False)
                 if not fullPostFilename:
-                    print('Unable to locate post ' + postUrl)
+                    print('Unable to locate post for newswire ' + postUrl)
                     ctr += 1
                     if ctr >= maxBlogsPerAccount:
                         break
@@ -840,7 +964,7 @@ def _addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
         for handle in dirs:
             if '@' not in handle:
                 continue
-            if 'inbox@' in handle:
+            if 'inbox@' in handle or 'news@' in handle:
                 continue
 
             nickname = handle.split('@')[0]

outbox.py

@@ -14,10 +14,12 @@ from posts import outboxMessageCreateWrap
 from posts import savePostToBox
 from posts import sendToFollowersThread
 from posts import sendToNamedAddresses
+from utils import getLocalNetworkAddresses
 from utils import getFullDomain
 from utils import removeIdEnding
 from utils import getDomainFromActor
 from utils import dangerousMarkup
+from utils import isFeaturedWriter
 from blocking import isBlockedDomain
 from blocking import outboxBlock
 from blocking import outboxUndoBlock
@@ -113,6 +115,23 @@ def postMessageToOutbox(messageJson: {}, postToNickname: str,
                           'Create does not have the "to" parameter ' +
                           str(messageJson))
             return False
+
+        # actor should be a string
+        if not isinstance(messageJson['actor'], str):
+            return False
+
+        # actor should look like a url
+        if '://' not in messageJson['actor'] or \
+           '.' not in messageJson['actor']:
+            return False
+
+        # sent by an actor on a local network address?
+        if not allowLocalNetworkAccess:
+            localNetworkPatternList = getLocalNetworkAddresses()
+            for localNetworkPattern in localNetworkPatternList:
+                if localNetworkPattern in messageJson['actor']:
+                    return False
+
         testDomain, testPort = getDomainFromActor(messageJson['actor'])
         testDomain = getFullDomain(testDomain, testPort)
         if isBlockedDomain(baseDir, testDomain):
@@ -211,14 +230,16 @@ def postMessageToOutbox(messageJson: {}, postToNickname: str,
         # save all instance blogs to the news actor
         if postToNickname != 'news' and outboxName == 'tlblogs':
             if '/' in savedFilename:
-                savedPostId = savedFilename.split('/')[-1]
+                if isFeaturedWriter(baseDir, postToNickname, domain):
-                blogsDir = baseDir + '/accounts/news@' + domain + '/tlblogs'
+                    savedPostId = savedFilename.split('/')[-1]
-                if not os.path.isdir(blogsDir):
+                    blogsDir = \
-                    os.mkdir(blogsDir)
+                        baseDir + '/accounts/news@' + domain + '/tlblogs'
-                copyfile(savedFilename, blogsDir + '/' + savedPostId)
+                    if not os.path.isdir(blogsDir):
-                inboxUpdateIndex('tlblogs', baseDir,
+                        os.mkdir(blogsDir)
-                                 'news@' + domain,
+                    copyfile(savedFilename, blogsDir + '/' + savedPostId)
-                                 savedFilename, debug)
+                    inboxUpdateIndex('tlblogs', baseDir,
+                                     'news@' + domain,
+                                     savedFilename, debug)
 
                 # clear the citations file if it exists
                 citationsFilename = \

person.py

@@ -40,6 +40,7 @@ from utils import loadJson
 from utils import saveJson
 from utils import setConfigParam
 from utils import getConfigParam
+from utils import refreshNewswire
 
 
 def generateRSAKey() -> (str, str):
@@ -915,6 +916,9 @@ def removeAccount(baseDir: str, nickname: str,
         os.remove(baseDir + '/wfdeactivated/' + handle + '.json')
     if os.path.isdir(baseDir + '/sharefilesdeactivated/' + nickname):
         shutil.rmtree(baseDir + '/sharefilesdeactivated/' + nickname)
+
+    refreshNewswire(baseDir)
+
     return True
 
 
@@ -944,6 +948,9 @@ def deactivateAccount(baseDir: str, nickname: str, domain: str) -> bool:
             os.mkdir(deactivatedSharefilesDir)
         shutil.move(baseDir + '/sharefiles/' + nickname,
                     deactivatedSharefilesDir + '/' + nickname)
+
+    refreshNewswire(baseDir)
+
     return os.path.isdir(deactivatedDir + '/' + nickname + '@' + domain)
 
 
@@ -970,6 +977,8 @@ def activateAccount(baseDir: str, nickname: str, domain: str) -> None:
             shutil.move(deactivatedSharefilesDir + '/' + nickname,
                         baseDir + '/sharefiles/' + nickname)
 
+    refreshNewswire(baseDir)
+
 
 def isPersonSnoozed(baseDir: str, nickname: str, domain: str,
                     snoozeActor: str) -> bool:

posts.py

@@ -30,6 +30,8 @@ from session import postJsonString
 from session import postImage
 from webfinger import webfingerHandle
 from httpsig import createSignedHeader
+from siteactive import siteIsActive
+from utils import removeInvalidChars
 from utils import fileLastModified
 from utils import isPublicPost
 from utils import hasUsersPath
@@ -38,7 +40,6 @@ from utils import getFullDomain
 from utils import getFollowersList
 from utils import isEvil
 from utils import removeIdEnding
-from utils import siteIsActive
 from utils import getCachedPostFilename
 from utils import getStatusNumber
 from utils import createPersonDir
@@ -823,7 +824,7 @@ def validContentWarning(cw: str) -> str:
     # so remove them
     if '#' in cw:
         cw = cw.replace('#', '').replace('  ', ' ')
-    return cw
+    return removeInvalidChars(cw)
 
 
 def _loadAutoCW(baseDir: str, nickname: str, domain: str) -> []:
@@ -880,6 +881,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
                     eventStatus=None, ticketUrl=None) -> {}:
     """Creates a message
     """
+    content = removeInvalidChars(content)
+
     subject = _addAutoCW(baseDir, nickname, domain, subject, content)
 
     if nickname != 'news':
@@ -924,7 +927,7 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
     sensitive = False
     summary = None
     if subject:
-        summary = validContentWarning(subject)
+        summary = removeInvalidChars(validContentWarning(subject))
         sensitive = True
 
     toRecipients = []
@@ -1047,6 +1050,8 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
     postObjectType = 'Note'
     if eventUUID:
         postObjectType = 'Event'
+    if isArticle:
+        postObjectType = 'Article'
 
     if not clientToServer:
         actorUrl = httpPrefix + '://' + domain + '/users/' + nickname
@@ -1389,10 +1394,22 @@ def createPublicPost(baseDir: str,
                      imageDescription: str,
                      inReplyTo=None, inReplyToAtomUri=None, subject=None,
                      schedulePost=False,
-                     eventDate=None, eventTime=None, location=None) -> {}:
+                     eventDate=None, eventTime=None, location=None,
+                     isArticle=False) -> {}:
     """Public post
     """
     domainFull = getFullDomain(domain, port)
+    isModerationReport = False
+    eventUUID = None
+    category = None
+    joinMode = None
+    endDate = None
+    endTime = None
+    maximumAttendeeCapacity = None
+    repliesModerationOption = None
+    anonymousParticipationEnabled = None
+    eventStatus = None
+    ticketUrl = None
     return _createPostBase(baseDir, nickname, domain, port,
                            'https://www.w3.org/ns/activitystreams#Public',
                            httpPrefix + '://' + domainFull + '/users/' +
@@ -1401,10 +1418,45 @@ def createPublicPost(baseDir: str,
                            clientToServer, commentsEnabled,
                            attachImageFilename, mediaType,
                            imageDescription,
-                           False, False, inReplyTo, inReplyToAtomUri, subject,
+                           isModerationReport, isArticle,
+                           inReplyTo, inReplyToAtomUri, subject,
                            schedulePost, eventDate, eventTime, location,
-                           None, None, None, None, None,
+                           eventUUID, category, joinMode, endDate, endTime,
-                           None, None, None, None, None)
+                           maximumAttendeeCapacity,
+                           repliesModerationOption,
+                           anonymousParticipationEnabled,
+                           eventStatus, ticketUrl)
+
+
+def _appendCitationsToBlogPost(baseDir: str,
+                               nickname: str, domain: str,
+                               blogJson: {}) -> None:
+    """Appends any citations to a new blog post
+    """
+    # append citations tags, stored in a file
+    citationsFilename = \
+        baseDir + '/accounts/' + \
+        nickname + '@' + domain + '/.citations.txt'
+    if not os.path.isfile(citationsFilename):
+        return
+    citationsSeparator = '#####'
+    with open(citationsFilename, "r") as f:
+        citations = f.readlines()
+        for line in citations:
+            if citationsSeparator not in line:
+                continue
+            sections = line.strip().split(citationsSeparator)
+            if len(sections) != 3:
+                continue
+            # dateStr = sections[0]
+            title = sections[1]
+            link = sections[2]
+            tagJson = {
+                "type": "Article",
+                "name": title,
+                "url": link
+            }
+            blogJson['object']['tag'].append(tagJson)
 
 
 def createBlogPost(baseDir: str,
@@ -1416,7 +1468,7 @@ def createBlogPost(baseDir: str,
                    inReplyTo=None, inReplyToAtomUri=None, subject=None,
                    schedulePost=False,
                    eventDate=None, eventTime=None, location=None) -> {}:
-    blog = \
+    blogJson = \
         createPublicPost(baseDir,
                          nickname, domain, port, httpPrefix,
                          content, followersOnly, saveToFile,
@@ -1425,34 +1477,11 @@ def createBlogPost(baseDir: str,
                          imageDescription,
                          inReplyTo, inReplyToAtomUri, subject,
                          schedulePost,
-                         eventDate, eventTime, location)
+                         eventDate, eventTime, location, True)
-    blog['object']['type'] = 'Article'
 
-    # append citations tags, stored in a file
+    _appendCitationsToBlogPost(baseDir, nickname, domain, blogJson)
-    citationsFilename = \
-        baseDir + '/accounts/' + \
-        nickname + '@' + domain + '/.citations.txt'
-    if os.path.isfile(citationsFilename):
-        citationsSeparator = '#####'
-        with open(citationsFilename, "r") as f:
-            citations = f.readlines()
-            for line in citations:
-                if citationsSeparator not in line:
-                    continue
-                sections = line.strip().split(citationsSeparator)
-                if len(sections) != 3:
-                    continue
-                # dateStr = sections[0]
-                title = sections[1]
-                link = sections[2]
-                tagJson = {
-                    "type": "Article",
-                    "name": title,
-                    "url": link
-                }
-                blog['object']['tag'].append(tagJson)
 
-    return blog
+    return blogJson
 
 
 def createNewsPost(baseDir: str,
@@ -1477,7 +1506,7 @@ def createNewsPost(baseDir: str,
                          imageDescription,
                          inReplyTo, inReplyToAtomUri, subject,
                          schedulePost,
-                         eventDate, eventTime, location)
+                         eventDate, eventTime, location, True)
     blog['object']['type'] = 'Article'
     return blog
 

session.py

@@ -53,6 +53,37 @@ def createSession(proxyType: str):
     return session
 
 
+def urlExists(session, url: str, timeoutSec=3,
+              httpPrefix='https', domain='testdomain') -> bool:
+    if not isinstance(url, str):
+        print('url: ' + str(url))
+        print('ERROR: urlExists failed, url should be a string')
+        return False
+    sessionParams = {}
+    sessionHeaders = {}
+    sessionHeaders['User-Agent'] = 'Epicyon/' + __version__
+    if domain:
+        sessionHeaders['User-Agent'] += \
+            '; +' + httpPrefix + '://' + domain + '/'
+    if not session:
+        print('WARN: urlExists failed, no session specified')
+        return True
+    try:
+        result = session.get(url, headers=sessionHeaders,
+                             params=sessionParams,
+                             timeout=timeoutSec)
+        if result:
+            if result.status_code == 200 or \
+               result.status_code == 304:
+                return True
+            else:
+                print('urlExists for ' + url + ' returned ' +
+                      str(result.status_code))
+    except BaseException:
+        pass
+    return False
+
+
 def getJson(session, url: str, headers: {}, params: {},
             version='1.2.0', httpPrefix='https',
             domain='testdomain') -> {}:
@@ -72,6 +103,7 @@ def getJson(session, url: str, headers: {}, params: {},
             '; +' + httpPrefix + '://' + domain + '/'
     if not session:
         print('WARN: getJson failed, no session specified for getJson')
+        return None
     try:
         result = session.get(url, headers=sessionHeaders, params=sessionParams)
         return result.json()

siteactive.py

@@ -0,0 +1,121 @@
+__filename__ = "siteactive.py"
+__author__ = "Bob Mottram"
+__credits__ = ["webchk"]
+__license__ = "AGPL3+"
+__version__ = "1.2.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@freedombone.net"
+__status__ = "Production"
+
+import http.client
+from urllib.parse import urlparse
+import ssl
+
+
+class Result:
+    """Holds result of an URL check.
+
+    The redirect attribute is a Result object that the URL was redirected to.
+
+    The sitemap_urls attribute will contain a list of Result object if url
+    is a sitemap file and http_response() was run with parse set to True.
+    """
+    def __init__(self, url):
+        self.url = url
+        self.status = 0
+        self.desc = ''
+        self.headers = None
+        self.latency = 0
+        self.content = ''
+        self.redirect = None
+        self.sitemap_urls = None
+
+    def __repr__(self):
+        if self.status == 0:
+            return '{} ... {}'.format(self.url, self.desc)
+        return '{} ... {} {} ({})'.format(
+            self.url, self.status, self.desc, self.latency
+        )
+
+    def fill_headers(self, headers):
+        """Takes a list of tuples and converts it a dictionary."""
+        self.headers = {h[0]: h[1] for h in headers}
+
+
+def _siteActiveParseUrl(url):
+    """Returns an object with properties representing
+
+    scheme:   URL scheme specifier
+    netloc:   Network location part
+    path:     Hierarchical path
+    params:   Parameters for last path element
+    query:    Query component
+    fragment: Fragment identifier
+    username: User name
+    password: Password
+    hostname: Host name (lower case)
+    port:     Port number as integer, if present
+    """
+    loc = urlparse(url)
+
+    # if the scheme (http, https ...) is not available urlparse wont work
+    if loc.scheme == "":
+        url = "http://" + url
+        loc = urlparse(url)
+    return loc
+
+
+def _siteACtiveHttpConnect(loc, timeout: int):
+    """Connects to the host and returns an HTTP or HTTPS connections."""
+    if loc.scheme == "https":
+        ssl_context = ssl.SSLContext()
+        return http.client.HTTPSConnection(
+            loc.netloc, context=ssl_context, timeout=timeout)
+    return http.client.HTTPConnection(loc.netloc, timeout=timeout)
+
+
+def _siteActiveHttpRequest(loc, timeout: int):
+    """Performs a HTTP request and return response in a Result object.
+    """
+    conn = _siteACtiveHttpConnect(loc, timeout)
+    method = 'HEAD'
+
+    conn.request(method, loc.path)
+    resp = conn.getresponse()
+
+    result = Result(loc.geturl())
+    result.status = resp.status
+    result.desc = resp.reason
+    result.fill_headers(resp.getheaders())
+
+    conn.close()
+    return result
+
+
+def siteIsActive(url: str, timeout=10) -> bool:
+    """Returns true if the current url is resolvable.
+    This can be used to check that an instance is online before
+    trying to send posts to it.
+    """
+    if not url.startswith('http'):
+        return False
+    if '.onion/' in url or '.i2p/' in url or \
+       url.endswith('.onion') or \
+       url.endswith('.i2p'):
+        # skip this check for onion and i2p
+        return True
+
+    loc = _siteActiveParseUrl(url)
+    result = Result(url=url)
+
+    try:
+        result = _siteActiveHttpRequest(loc, timeout)
+
+        if 400 <= result.status < 500:
+            return result
+
+        return True
+
+    except BaseException:
+        pass
+    return False

tests.py

@@ -38,7 +38,7 @@ from utils import getFullDomain
 from utils import validNickname
 from utils import firstParagraphFromString
 from utils import removeIdEnding
-from utils import siteIsActive
+from siteactive import siteIsActive
 from utils import updateRecentPostsCache
 from utils import followPerson
 from utils import getNicknameFromActor
@@ -325,8 +325,10 @@ def createServerAlice(path: str, domain: str, port: int,
     sendThreadsTimeoutMins = 30
     maxFollowers = 10
     verifyAllSignatures = True
+    brochMode = False
     print('Server running: Alice')
-    runDaemon(verifyAllSignatures,
+    runDaemon(brochMode,
+              verifyAllSignatures,
               sendThreadsTimeoutMins,
               dormantMonths, maxNewswirePosts,
               allowLocalNetworkAccess,
@@ -420,8 +422,10 @@ def createServerBob(path: str, domain: str, port: int,
     sendThreadsTimeoutMins = 30
     maxFollowers = 10
     verifyAllSignatures = True
+    brochMode = False
     print('Server running: Bob')
-    runDaemon(verifyAllSignatures,
+    runDaemon(brochMode,
+              verifyAllSignatures,
               sendThreadsTimeoutMins,
               dormantMonths, maxNewswirePosts,
               allowLocalNetworkAccess,
@@ -469,8 +473,10 @@ def createServerEve(path: str, domain: str, port: int, federationList: [],
     sendThreadsTimeoutMins = 30
     maxFollowers = 10
     verifyAllSignatures = True
+    brochMode = False
     print('Server running: Eve')
-    runDaemon(verifyAllSignatures,
+    runDaemon(brochMode,
+              verifyAllSignatures,
               sendThreadsTimeoutMins,
               dormantMonths, maxNewswirePosts,
               allowLocalNetworkAccess,
@@ -2067,6 +2073,7 @@ def testJsonld():
 
 def testSiteIsActive():
     print('testSiteIsActive')
+    assert(siteIsActive('https://archive.org'))
     assert(siteIsActive('https://mastodon.social'))
     assert(not siteIsActive('https://notarealwebsite.a.b.c'))
 
@@ -2818,7 +2825,8 @@ def testFunctions():
         'createServerBob',
         'createServerEve',
         'E2EEremoveDevice',
-        'setOrganizationScheme'
+        'setOrganizationScheme',
+        'fill_headers'
     ]
     excludeImports = [
         'link',

theme/default/theme.json

@@ -1,4 +1,6 @@
 {
+    "post-separator-margin-top": "10px",
+    "post-separator-margin-bottom": "10px",
     "newswire-publish-icon": "True",
     "full-width-timeline-buttons": "False",
     "icons-as-buttons": "False",

theme/indymediamodern/theme.json

@@ -6,7 +6,7 @@
     "header-bg-color": "#efefef",
     "verticals-width": "10px",
     "newswire-publish-icon": "False",
-    "line-spacing-newswire": "150%",
+    "line-spacing-newswire": "180%",
     "full-width-timeline-buttons": "False",
     "icons-as-buttons": "True",
     "rss-icon-at-top": "False",

theme/night/theme.json

@@ -60,7 +60,7 @@
     "event-color": "#0481f5",
     "event-background": "#00014a",
     "quote-right-margin": "0",
-    "line-spacing": "150%",
+    "line-spacing": "180%",
     "header-font": "'solidaric'",
     "*font-family": "'solidaric'",
     "*src": "url('./fonts/solidaric.woff2') format('woff2')",

theme/rc3/theme.json

@@ -84,7 +84,7 @@
     "event-color": "#0481f5",
     "event-background": "#00014a",
     "quote-right-margin": "0",    
-    "line-spacing": "150%",
+    "line-spacing": "180%",
     "*font-family": "'Montserrat-Regular'",
     "*src": "url('./fonts/Montserrat-Regular.ttf') format('truetype')",
     "**font-family": "'Orbitron'",

theme/solidaric/theme.json

@@ -80,7 +80,7 @@
     "title-background": "#ccc",
     "gallery-text-color": "#2d2c37",
     "quote-right-margin": "0",
-    "line-spacing": "150%",
+    "line-spacing": "180%",
     "header-font": "'solidaric'",
     "*font-family": "'solidaric'",
     "*src": "url('./fonts/solidaric.woff2') format('woff2')",

translations/ar.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "تخطي إلى الجدول الزمني",
     "Skip to Newswire": "انتقل إلى Newswire",
     "Skip to Links": "تخطي إلى روابط الويب",
-    "Publish a blog article": "نشر مقال بلوق"
+    "Publish a blog article": "نشر مقال بلوق",
+    "Featured writer": "كاتب متميز",
+    "Broch mode": "وضع الكتيب"
 }

translations/ca.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Ves a la cronologia",
     "Skip to Newswire": "Vés a Newswire",
     "Skip to Links": "Vés als enllaços web",
-    "Publish a blog article": "Publicar un article del bloc"
+    "Publish a blog article": "Publicar un article del bloc",
+    "Featured writer": "Escriptor destacat",
+    "Broch mode": "Mode Broch"
 }

translations/cy.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Neidio i'r llinell amser",
     "Skip to Newswire": "Neidio i Newswire",
     "Skip to Links": "Neidio i Dolenni Gwe",
-    "Publish a blog article": "Cyhoeddi erthygl blog"
+    "Publish a blog article": "Cyhoeddi erthygl blog",
+    "Featured writer": "Awdur dan sylw",
+    "Broch mode": "Modd Broch"
 }

translations/de.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Zur Zeitleiste springen",
     "Skip to Newswire": "Springe zu Newswire",
     "Skip to Links": "Springe zu Weblinks",
-    "Publish a blog article": "Veröffentlichen Sie einen Blog-Artikel"
+    "Publish a blog article": "Veröffentlichen Sie einen Blog-Artikel",
+    "Featured writer": "Ausgewählter Schriftsteller",
+    "Broch mode": "Broch-Modus"
 }

translations/en.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Skip to timeline",
     "Skip to Newswire": "Skip to Newswire",
     "Skip to Links": "Skip to Links",
-    "Publish a blog article": "Publish a blog article"
+    "Publish a blog article": "Publish a blog article",
+    "Featured writer": "Featured writer",
+    "Broch mode": "Broch mode"
 }

translations/es.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Saltar a la línea de tiempo",
     "Skip to Newswire": "Saltar a Newswire",
     "Skip to Links": "Saltar a enlaces web",
-    "Publish a blog article": "Publica un artículo de blog"
+    "Publish a blog article": "Publica un artículo de blog",
+    "Featured writer": "Escritora destacada",
+    "Broch mode": "Modo broche"
 }

translations/fr.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Passer à la chronologie",
     "Skip to Newswire": "Passer à Newswire",
     "Skip to Links": "Passer aux liens Web",
-    "Publish a blog article": "Publier un article de blog"
+    "Publish a blog article": "Publier un article de blog",
+    "Featured writer": "Écrivain en vedette",
+    "Broch mode": "Mode Broch"
 }

translations/ga.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Scipeáil chuig an amlíne",
     "Skip to Newswire": "Scipeáil chuig Newswire",
     "Skip to Links": "Scipeáil chuig Naisc Ghréasáin",
-    "Publish a blog article": "Foilsigh alt blagála"
+    "Publish a blog article": "Foilsigh alt blagála",
+    "Featured writer": "Scríbhneoir mór le rá",
+    "Broch mode": "Modh broch"
 }

translations/hi.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "टाइमलाइन पर जाएं",
     "Skip to Newswire": "Newswire पर जाएं",
     "Skip to Links": "वेब लिंक पर जाएं",
-    "Publish a blog article": "एक ब्लॉग लेख प्रकाशित करें"
+    "Publish a blog article": "एक ब्लॉग लेख प्रकाशित करें",
+    "Featured writer": "फीचर्ड लेखक",
+    "Broch mode": "ब्रोच मोड"
 }

translations/it.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Passa alla sequenza temporale",
     "Skip to Newswire": "Passa a Newswire",
     "Skip to Links": "Passa a collegamenti Web",
-    "Publish a blog article": "Pubblica un articolo sul blog"
+    "Publish a blog article": "Pubblica un articolo sul blog",
+    "Featured writer": "Scrittore in primo piano",
+    "Broch mode": "Modalità Broch"
 }

translations/ja.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "タイムラインにスキップ",
     "Skip to Newswire": "Newswireにスキップ",
     "Skip to Links": "Webリンクにスキップ",
-    "Publish a blog article": "ブログ記事を公開する"
+    "Publish a blog article": "ブログ記事を公開する",
+    "Featured writer": "注目の作家",
+    "Broch mode": "ブロッホモード"
 }

translations/oc.json

@@ -363,5 +363,7 @@
     "Skip to timeline": "Skip to timeline",
     "Skip to Newswire": "Skip to Newswire",
     "Skip to Links": "Skip to Links",
-    "Publish a blog article": "Publish a blog article"
+    "Publish a blog article": "Publish a blog article",
+    "Featured writer": "Featured writer",
+    "Broch mode": "Broch mode"
 }

translations/pt.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Pular para a linha do tempo",
     "Skip to Newswire": "Pular para Newswire",
     "Skip to Links": "Pular para links da web",
-    "Publish a blog article": "Publique um artigo de blog"
+    "Publish a blog article": "Publique um artigo de blog",
+    "Featured writer": "Escritor em destaque",
+    "Broch mode": "Modo broch"
 }

translations/ru.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "Перейти к временной шкале",
     "Skip to Newswire": "Перейти к ленте новостей",
     "Skip to Links": "Перейти к веб-ссылкам",
-    "Publish a blog article": "Опубликовать статью в блоге"
+    "Publish a blog article": "Опубликовать статью в блоге",
+    "Featured writer": "Избранный писатель",
+    "Broch mode": "Брош режим"
 }

translations/zh.json

@@ -367,5 +367,7 @@
     "Skip to timeline": "跳到时间线",
     "Skip to Newswire": "跳到新闻专线",
     "Skip to Links": "跳到网页链接",
-    "Publish a blog article": "发布博客文章"
+    "Publish a blog article": "发布博客文章",
+    "Featured writer": "特色作家",
+    "Broch mode": "断点模式"
 }

utils.py

@@ -11,9 +11,6 @@ import time
 import shutil
 import datetime
 import json
-from socket import error as SocketError
-import errno
-import urllib.request
 import idna
 from pprint import pprint
 from calendar import monthrange
@@ -21,6 +18,34 @@ from followingCalendar import addPersonToCalendar
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 
+# posts containing these strings will always get screened out,
+# both incoming and outgoing.
+# Could include dubious clacks or admin dogwhistles
+invalidCharacters = (
+    '卐', '卍', '࿕', '࿖', '࿗', '࿘'
+)
+
+
+def isFeaturedWriter(baseDir: str, nickname: str, domain: str) -> bool:
+    """Is the given account a featured writer, appearing in the features
+    timeline on news instances?
+    """
+    featuresBlockedFilename = \
+        baseDir + '/accounts/' + \
+        nickname + '@' + domain + '/.nofeatures'
+    return not os.path.isfile(featuresBlockedFilename)
+
+
+def refreshNewswire(baseDir: str):
+    """Causes the newswire to be updates after a change to user accounts
+    """
+    refreshNewswireFilename = baseDir + '/accounts/.refresh_newswire'
+    if os.path.isfile(refreshNewswireFilename):
+        return
+    refreshFile = open(refreshNewswireFilename, 'w+')
+    refreshFile.write('\n')
+    refreshFile.close()
+
 
 def getSHA256(msg: str):
     """Returns a SHA256 hash of the given string
@@ -517,17 +542,23 @@ def isEvil(domain: str) -> bool:
 
 def containsInvalidChars(jsonStr: str) -> bool:
     """Does the given json string contain invalid characters?
-    e.g. dubious clacks/admin dogwhistles
     """
-    invalidStrings = {
+    for isInvalid in invalidCharacters:
-        '卐', '卍', '࿕', '࿖', '࿗', '࿘'
-    }
-    for isInvalid in invalidStrings:
         if isInvalid in jsonStr:
             return True
     return False
 
 
+def removeInvalidChars(text: str) -> str:
+    """Removes any invalid characters from a string
+    """
+    for isInvalid in invalidCharacters:
+        if isInvalid not in text:
+            continue
+        text = text.replace(isInvalid, '')
+    return text
+
+
 def createPersonDir(nickname: str, domain: str, baseDir: str,
                     dirname: str) -> str:
     """Create a directory for a person
@@ -574,6 +605,12 @@ def urlPermitted(url: str, federationList: []):
     return False
 
 
+def getLocalNetworkAddresses() -> []:
+    """Returns patterns for local network address detection
+    """
+    return ('localhost', '127.0.', '192.168', '10.0.')
+
+
 def dangerousMarkup(content: str, allowLocalNetworkAccess: bool) -> bool:
     """Returns true if the given content contains dangerous html markup
     """
@@ -584,7 +621,7 @@ def dangerousMarkup(content: str, allowLocalNetworkAccess: bool) -> bool:
     contentSections = content.split('<')
     invalidPartials = ()
     if not allowLocalNetworkAccess:
-        invalidPartials = ('localhost', '127.0.', '192.168', '10.0.')
+        invalidPartials = getLocalNetworkAddresses()
     invalidStrings = ('script', 'canvas', 'style', 'abbr',
                       'frame', 'iframe', 'html', 'body',
                       'hr', 'allow-popups', 'allow-scripts')
@@ -1841,28 +1878,6 @@ def updateAnnounceCollection(recentPostsCache: {},
         saveJson(postJsonObject, postFilename)
 
 
-def siteIsActive(url: str) -> bool:
-    """Returns true if the current url is resolvable.
-    This can be used to check that an instance is online before
-    trying to send posts to it.
-    """
-    if not url.startswith('http'):
-        return False
-    if '.onion/' in url or '.i2p/' in url or \
-       url.endswith('.onion') or \
-       url.endswith('.i2p'):
-        # skip this check for onion and i2p
-        return True
-    try:
-        req = urllib.request.Request(url)
-        urllib.request.urlopen(req, timeout=10)  # nosec
-        return True
-    except SocketError as e:
-        if e.errno == errno.ECONNRESET:
-            print('WARN: connection was reset during siteIsActive')
-    return False
-
-
 def weekDayOfMonthStart(monthNumber: int, year: int) -> int:
     """Gets the day number of the first day of the month
     1=sun, 7=sat

webapp_calendar.py

@@ -21,6 +21,8 @@ from happening import getCalendarEvents
 from webapp_utils import htmlHeaderWithExternalStyle
 from webapp_utils import htmlFooter
 from webapp_utils import getAltPath
+from webapp_utils import htmlHideFromScreenReader
+from webapp_utils import htmlKeyboardNavigation
 
 
 def htmlCalendarDeleteConfirm(cssCache: {}, translate: {}, baseDir: str,
@@ -200,7 +202,8 @@ def _htmlCalendarDay(cssCache: {}, translate: {},
 
 def htmlCalendar(cssCache: {}, translate: {},
                  baseDir: str, path: str,
-                 httpPrefix: str, domainFull: str) -> str:
+                 httpPrefix: str, domainFull: str,
+                 textModeBanner: str) -> str:
     """Show the calendar for a person
     """
     domain = domainFull
@@ -297,8 +300,10 @@ def htmlCalendar(cssCache: {}, translate: {},
 
     instanceTitle = \
         getConfigParam(baseDir, 'instanceTitle')
-    calendarStr = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
+    headerStr = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
-    calendarStr += '<main><table class="calendar">\n'
+
+    # the main graphical calendar as a table
+    calendarStr = '<main><table class="calendar">\n'
     calendarStr += '<caption class="calendar__banner--month">\n'
     calendarStr += \
         '  <a href="' + calActor + '/calendar?year=' + str(prevYear) + \
@@ -320,24 +325,30 @@ def htmlCalendar(cssCache: {}, translate: {},
     calendarStr += '</caption>\n'
     calendarStr += '<thead>\n'
     calendarStr += '<tr>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Sun'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Mon'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Tue'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Wed'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Thu'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Fri'] + '</th>\n'
-    calendarStr += '  <th class="calendar__day__header">' + \
+    calendarStr += '  <th scope="col" class="calendar__day__header">' + \
         translate['Sat'] + '</th>\n'
     calendarStr += '</tr>\n'
     calendarStr += '</thead>\n'
     calendarStr += '<tbody>\n'
 
+    # beginning of the links used for accessibility
+    navLinks = {}
+    timelineLinkStr = htmlHideFromScreenReader('🏠') + ' ' + \
+        translate['Switch to timeline view']
+    navLinks[timelineLinkStr] = calActor + '/inbox'
+
     dayOfMonth = 0
     dow = weekDayOfMonthStart(monthNumber, year)
     for weekOfMonth in range(1, 7):
@@ -358,8 +369,15 @@ def htmlCalendar(cssCache: {}, translate: {},
                     url = calActor + '/calendar?year=' + \
                         str(year) + '?month=' + \
                         str(monthNumber) + '?day=' + str(dayOfMonth)
-                    dayLink = '<a href="' + url + '">' + \
+                    dayDescription = monthName + ' ' + str(dayOfMonth)
+                    dayLink = '<a href="' + url + '" ' + \
+                        'title="' + dayDescription + '">' + \
                         str(dayOfMonth) + '</a>'
+                    # accessibility menu links
+                    menuOptionStr = \
+                        htmlHideFromScreenReader('📅') + ' ' + \
+                        dayDescription
+                    navLinks[menuOptionStr] = url
                     # there are events for this day
                     if not isToday:
                         calendarStr += \
@@ -387,5 +405,17 @@ def htmlCalendar(cssCache: {}, translate: {},
 
     calendarStr += '</tbody>\n'
     calendarStr += '</table></main>\n'
-    calendarStr += htmlFooter()
+
-    return calendarStr
+    # end of the links used for accessibility
+    nextMonthStr = \
+        htmlHideFromScreenReader('→') + ' ' + translate['Next month']
+    navLinks[nextMonthStr] = calActor + '/calendar?year=' + str(nextYear) + \
+        '?month=' + str(nextMonthNumber)
+    prevMonthStr = \
+        htmlHideFromScreenReader('←') + ' ' + translate['Previous month']
+    navLinks[prevMonthStr] = calActor + '/calendar?year=' + str(prevYear) + \
+        '?month=' + str(prevMonthNumber)
+    screenReaderCal = \
+        htmlKeyboardNavigation(textModeBanner, navLinks, monthName)
+
+    return headerStr + screenReaderCal + calendarStr + htmlFooter()

webapp_hashtagswarm.py

@@ -203,8 +203,12 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str:
                             categoryStr = \
                                 getHashtagCategory(baseDir, hashTagName)
                             if len(categoryStr) < maxTagLength:
-                                if categoryStr not in categorySwarm:
+                                if '#' not in categoryStr and \
-                                    categorySwarm.append(categoryStr)
+                                   '&' not in categoryStr and \
+                                   '"' not in categoryStr and \
+                                   "'" not in categoryStr:
+                                    if categoryStr not in categorySwarm:
+                                        categorySwarm.append(categoryStr)
                         break
         break
 

webapp_person_options.py

@@ -17,6 +17,7 @@ from utils import isDormant
 from utils import removeHtml
 from utils import getDomainFromActor
 from utils import getNicknameFromActor
+from utils import isFeaturedWriter
 from blocking import isBlocked
 from follow import isFollowerOfPerson
 from follow import isFollowingActor
@@ -24,6 +25,7 @@ from followingCalendar import receivingCalendarEvents
 from webapp_utils import htmlHeaderWithExternalStyle
 from webapp_utils import htmlFooter
 from webapp_utils import getBrokenLinkSubstitute
+from webapp_utils import htmlKeyboardNavigation
 
 
 def htmlPersonOptions(defaultTimeline: str,
@@ -49,7 +51,9 @@ def htmlPersonOptions(defaultTimeline: str,
                       backToPath: str,
                       lockedAccount: bool,
                       movedTo: str,
-                      alsoKnownAs: []) -> str:
+                      alsoKnownAs: [],
+                      textModeBanner: str,
+                      newsInstance: bool) -> str:
     """Show options for a person: view/follow/block/report
     """
     optionsDomain, optionsPort = getDomainFromActor(optionsActor)
@@ -108,12 +112,13 @@ def htmlPersonOptions(defaultTimeline: str,
     if donateUrl:
         donateStr = \
             '    <a href="' + donateUrl + \
-            '"><button class="button" name="submitDonate">' + \
+            ' tabindex="-1""><button class="button" name="submitDonate">' + \
             translate['Donate'] + '</button></a>\n'
 
     instanceTitle = \
         getConfigParam(baseDir, 'instanceTitle')
     optionsStr = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
+    optionsStr += htmlKeyboardNavigation(textModeBanner, {})
     optionsStr += '<br><br>\n'
     optionsStr += '<div class="options">\n'
     optionsStr += '  <div class="optionsAvatar">\n'
@@ -284,6 +289,24 @@ def htmlPersonOptions(defaultTimeline: str,
             checkboxStr = checkboxStr.replace(' checked>', '>')
         optionsStr += checkboxStr
 
+    # checkbox for permission to post to featured articles
+    if newsInstance and optionsDomainFull == domainFull:
+        adminNickname = getConfigParam(baseDir, 'admin')
+        if (nickname == adminNickname or
+            (isModerator(baseDir, nickname) and
+             not isModerator(baseDir, optionsNickname))):
+            checkboxStr = \
+                '    <input type="checkbox" ' + \
+                'class="profilecheckbox" name="postsToFeatures" checked> ' + \
+                translate['Featured writer'] + \
+                '\n    <button type="submit" class="buttonsmall" ' + \
+                'name="submitPostToFeatures">' + \
+                translate['Submit'] + '</button><br>\n'
+            if not isFeaturedWriter(baseDir, optionsNickname,
+                                    optionsDomain):
+                checkboxStr = checkboxStr.replace(' checked>', '>')
+            optionsStr += checkboxStr
+
     optionsStr += optionsLinkStr
     backPath = '/'
     if nickname:

webapp_post.py

@@ -327,9 +327,13 @@ def _getEditIconHtml(baseDir: str, nickname: str, domainFull: str,
     """
     editStr = ''
     actor = postJsonObject['actor']
-    if (actor.endswith(domainFull + '/users/' + nickname) or
+    # This should either be a post which you created,
+    # or it could be generated from the newswire (see
+    # _addBlogsToNewswire) in which case anyone with
+    # editor status should be able to alter it
+    if (actor.endswith('/' + domainFull + '/users/' + nickname) or
         (isEditor(baseDir, nickname) and
-         actor.endswith(domainFull + '/users/news'))):
+         actor.endswith('/' + domainFull + '/users/news'))):
 
         postId = postJsonObject['object']['id']
 

webapp_profile.py

@@ -1278,6 +1278,16 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str,
                     '      <input type="checkbox" class="profilecheckbox" ' + \
                     'name="verifyallsignatures"> ' + \
                     translate['Verify all signatures'] + '<br>\n'
+            if getConfigParam(baseDir, "brochMode"):
+                instanceStr += \
+                    '      <input type="checkbox" class="profilecheckbox" ' + \
+                    'name="brochMode" checked> ' + \
+                    translate['Broch mode'] + '<br>\n'
+            else:
+                instanceStr += \
+                    '      <input type="checkbox" class="profilecheckbox" ' + \
+                    'name="brochMode"> ' + \
+                    translate['Broch mode'] + '<br>\n'
             instanceStr += '</div>'
 
             moderators = ''
@@ -1745,6 +1755,10 @@ def _individualFollowAsHtml(translate: {},
     if avatarUrl2:
         avatarUrl = avatarUrl2
     if displayName:
+        displayName = \
+            addEmojiToDisplayName(baseDir, httpPrefix,
+                                  actorNickname, domain,
+                                  displayName, False)
         titleStr = displayName
 
     if dormant:

webapp_timeline.py

@@ -416,23 +416,23 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str,
         translate['Mod']
     navLinks = {
         menuProfile: '/users/' + nickname,
-        menuInbox: usersPath + '/inbox#timeline',
+        menuInbox: usersPath + '/inbox#timelineposts',
         menuSearch: usersPath + '/search',
         menuNewPost: usersPath + '/newpost',
         menuCalendar: usersPath + '/calendar',
-        menuDM: usersPath + '/dm#timeline',
+        menuDM: usersPath + '/dm#timelineposts',
-        menuReplies: usersPath + '/tlreplies#timeline',
+        menuReplies: usersPath + '/tlreplies#timelineposts',
-        menuOutbox: usersPath + '/inbox#timeline',
+        menuOutbox: usersPath + '/inbox#timelineposts',
-        menuBookmarks: usersPath + '/tlbookmarks#timeline',
+        menuBookmarks: usersPath + '/tlbookmarks#timelineposts',
-        menuShares: usersPath + '/tlshares#timeline',
+        menuShares: usersPath + '/tlshares#timelineposts',
-        menuBlogs: usersPath + '/tlblogs#timeline',
+        menuBlogs: usersPath + '/tlblogs#timelineposts',
-        # menuEvents: usersPath + '/tlevents#timeline',
+        # menuEvents: usersPath + '/tlevents#timelineposts',
-        menuNewswire: '#newswire',
+        menuNewswire: usersPath + '/newswiremobile',
-        menuLinks: '#links'
+        menuLinks: usersPath + '/linksmobile'
     }
     if moderator:
         navLinks[menuModeration] = usersPath + '/moderation#modtimeline'
-    tlStr += htmlKeyboardNavigation(textModeBanner, navLinks,
+    tlStr += htmlKeyboardNavigation(textModeBanner, navLinks, None,
                                     usersPath, translate, followApprovals)
 
     # banner and row of buttons
@@ -502,7 +502,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str,
                                   calendarImage, followApprovals,
                                   iconsAsButtons)
 
-    tlStr += '  <div id="timeline" class="timeline-posts">\n'
+    tlStr += '  <div id="timelineposts" class="timeline-posts">\n'
 
     # second row of buttons for moderator actions
     if moderator and boxName == 'moderation':

webapp_utils.py

@@ -887,26 +887,31 @@ def htmlHideFromScreenReader(htmlStr: str) -> str:
 
 
 def htmlKeyboardNavigation(banner: str, links: {},
+                           subHeading=None,
                            usersPath=None, translate=None,
                            followApprovals=False) -> str:
     """Given a set of links return the html for keyboard navigation
     """
-    htmlStr = '<div class="transparent"><ul>'
+    htmlStr = '<div class="transparent"><ul>\n'
 
     if banner:
-        htmlStr += '<pre aria-label="">' + banner + '<br><br></pre>'
+        htmlStr += '<pre aria-label="">\n' + banner + '\n<br><br></pre>\n'
+
+    if subHeading:
+        htmlStr += '<strong><label class="transparent">' + \
+            subHeading + '</label></strong><br>\n'
 
     # show new follower approvals
     if usersPath and translate and followApprovals:
         htmlStr += '<strong><label class="transparent">' + \
             '<a href="' + usersPath + '/followers#timeline">' + \
             translate['Approve follow requests'] + '</a>' + \
-            '</label></strong><br><br>'
+            '</label></strong><br><br>\n'
 
     # show the list of links
     for title, url in links.items():
         htmlStr += '<li><label class="transparent">' + \
             '<a href="' + str(url) + '">' + \
-            str(title) + '</a></label></li>'
+            str(title) + '</a></label></li>\n'
-    htmlStr += '</ul></div>'
+    htmlStr += '</ul></div>\n'
     return htmlStr

website/EN/index.html

@@ -1,6 +1,9 @@
 <!DOCTYPE html>
 <html lang="en">
   <meta charset="utf-8">
+  <meta name="description" content="ActivityPub server written in Python, HTML and CSS, and suitable for self-hosting on single board computers">
+  <meta name="keywords" content="ActivityPub, Fediverse, Python, HTML, CSS">
+  <meta name="author" content="Bob Mottram">
   <style>
     @charset "UTF-8";
 
@@ -1141,6 +1144,9 @@
     }
 
   </style>
+  <head>
+    <title>Epicyon ActivityPub server</title>
+  </head>
   <body>
     <div class="timeline-banner"></div>
     <center>