Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon into main

README_commandline.md

@@ -390,3 +390,15 @@ To remove a shared item:
 ``` bash
 python3 epicyon.py --undoItemName "spanner" --nickname [yournick] --domain [yourdomain] --password [c2s password]
 ```
+
+## Speaking your inbox
+
+It is possible to use text-to-speech to read your inbox as posts arrive. This can be useful if you are not looking at a screen but want to stay ambiently informed of what's happening.
+
+On Debian based systems you will need to have the **python3-espeak** package installed.
+
+``` bash
+python3 epicyon.py --speaker yournickname@yourdomain --password [yourpassword]
+```
+
+This will then stay running and incoming posts will be announced as they arrive.

content.py

@@ -995,6 +995,11 @@ def extractTextFieldsInPOST(postBytes, boundary, debug: bool,
 
     messageFields = messageFields.split(boundary)
     fields = {}
+    fieldsWithSemicolonAllowed = (
+        'message', 'bio', 'autoCW', 'password', 'passwordconfirm',
+        'instanceDescription', 'instanceDescriptionShort',
+        'subject', 'location', 'imageDescription'
+    )
     # examine each section of the POST, separated by the boundary
     for f in messageFields:
         if f == '--':
@@ -1007,7 +1012,8 @@ def extractTextFieldsInPOST(postBytes, boundary, debug: bool,
         postKey = postStr.split('"', 1)[0]
         postValueStr = postStr.split('"', 1)[1]
         if ';' in postValueStr:
-            if postKey != 'message':
+            if postKey not in fieldsWithSemicolonAllowed and \
+               not postKey.startswith('edited'):
                 continue
         if '\r\n' not in postValueStr:
             continue

daemon.py

@@ -3315,8 +3315,8 @@ class PubServer(BaseHTTPRequestHandler):
                 return
 
             linksFilename = baseDir + '/accounts/links.txt'
-            aboutFilename = baseDir + '/accounts/about.txt'
+            aboutFilename = baseDir + '/accounts/about.md'
-            TOSFilename = baseDir + '/accounts/tos.txt'
+            TOSFilename = baseDir + '/accounts/tos.md'
 
             # extract all of the text fields into a dict
             fields = \
@@ -5242,6 +5242,28 @@ class PubServer(BaseHTTPRequestHandler):
             print('favicon not sent: ' + callingDomain)
         self._404()
 
+    def _getSpeaker(self, callingDomain: str, path: str,
+                    baseDir: str, domain: str, debug: bool) -> None:
+        """Returns the speaker file used for TTS and
+        accessed via c2s
+        """
+        nickname = path.split('/users/')[1]
+        if '/' in nickname:
+            nickname = nickname.split('/')[0]
+        speakerFilename = \
+            baseDir + '/accounts/' + nickname + '@' + domain + '/speaker.json'
+        if not os.path.isfile(speakerFilename):
+            self._404()
+            return
+
+        speakerJson = loadJson(speakerFilename)
+        msg = json.dumps(speakerJson,
+                         ensure_ascii=False).encode('utf-8')
+        msglen = len(msg)
+        self._set_headers('application/json', msglen,
+                          None, callingDomain)
+        self._write(msg)
+
     def _getFonts(self, callingDomain: str, path: str,
                   baseDir: str, debug: bool,
                   GETstartTime, GETtimings: {}) -> None:
@@ -10454,6 +10476,16 @@ class PubServer(BaseHTTPRequestHandler):
         if '/users/' in self.path:
             usersInPath = True
 
+        # authorized endpoint used for TTS of posts
+        # arriving in your inbox
+        if authorized and usersInPath and \
+           self.path.endswith('/speaker'):
+            self._getSpeaker(callingDomain, self.path,
+                             self.server.baseDir,
+                             self.server.domain,
+                             self.server.debug)
+            return
+
         # redirect to the welcome screen
         if htmlGET and authorized and usersInPath and \
            '/welcome' not in self.path:

default_about.md

@@ -0,0 +1,9 @@
+# About this Instance
+### Origin Story
+How your instance began.
+
+### Lore
+Customs and rituals.
+
+### Epic Tales
+Heroic deeds and dastardly foes.

default_about.txt

@@ -1,13 +0,0 @@
-<h1>About this Instance</h1>
-
-<h3>Origin Story</h3>
-
-<p>How your instance began.</p>
-
-<h3>Lore</h3>
-
-<p>Customs and rituals.</p>
-
-<h3>Epic Tales</h3>
-
-<p>Heroic deeds and dastardly foes.</p>

default_tos.md

@@ -0,0 +1,44 @@
+# Terms of Service
+### Data Collected
+Your username and a hash of your password, any posts you make and a list of accounts which you follow. The admin of the site does not know your password and it is not stored in plaintext anywhere.
+
+There is a quota on the number of posts retained by this instance for each account. Older posts will be removed when the limit is reached. Anything you post here should be considered ephemeral and you should keep a separate personal copy of them if you wish to retain a permanent archive.
+
+No IP addresses are logged.
+
+Posts can be removed on request if there is sufficient justification, but the nature of ActivityPub means that deletion of data federated to other instances cannot be guaranteed.
+
+### Content Policy
+This instance will not host content containing sexism, racism, casteism, homophobia, transphobia, misogyny, antisemitism or other forms of bigotry or discrimination on the basis of nationality or immigration status. Claims that transgressions of this type were intended to be "ironic" will be treated as a terms of service violation.
+
+Even if not conspicuously discriminatory, expressions of support for organizations with discrminatory agendas are not permitted on this instance. These include, but are not limited to, racial supremacist groups, the redpill/incel movement and anti-LGBT or anti-immigrant campaigns.
+
+Depictions of injury, death or medical procedures are not permitted.
+
+Violent or abusive content will be subject to moderation and is likely to be removed.
+
+Content of a sexual nature may be published providing that only consenting adults (aged 18 or over) are depicted and an appropriate content warning message is added. Posting sexual content without a content warning is a terms of service violation. Sexual content is defined both as photographs of real people and also artistic or fictional depictions, edited/generated photos or narratives.
+
+Moderators rely upon your reports. Don't assume that something of concern has already been reported. It's better for there to be duplicate reports than for something potentially damaging to go unreported.
+
+Content found to be non-compliant with this policy will be removed and any accounts on this instance producing, repeating or linking to such content will be deleted typically without prior notification.
+
+### Federation Policy
+In a proactive effort to avoid the classic fate of *"embrace, extend, extinguish"* this system will block any instance launched, acquired or funded by Alphabet, Facebook, Twitter, Microsoft, Apple, Amazon, Elsevier or other monopolistic Silicon Valley companies.
+
+This system will not federate with instances whose moderation policy is incompatible with the content policy described above. If an instance lacks a moderation policy, or refuses to enforce one, it will be assumed to be incompatible.
+
+### Use of User Generated Content for Research
+Data may not be "scraped" or otherwise obtained from this instance and used for academic research or cited within research publications without the prior written permission of the administrator. Financial remedy will be sought through the courts from any researcher publishing data obtained from this instance without consent.
+
+### Commercial Use
+Commercial use of original content on this instance is strictly forbidden without the prior written permission of individual account holders. The instance administrator does not hold copyright on any original content posted by account holders. Publication or federation of content does not imply permission for commercial use.
+
+Commercial use includes the harvesting of data to create products which are then sold, such as statistics, business reports or machine learning models.
+
+### Copyrights
+Epicyon is licensed under [GNU AGPL version 3](https://www.gnu.org/licenses/agpl-3.0-standalone.html)
+
+Emojis designed by [OpenMoji](https://openmoji.org) – the open-source emoji and icon project. License: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0)
+
+Blob Cat Emoji and Meowmoji were made by Nitro Blob Hub, licensed under [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0)

default_tos.txt

@@ -1,51 +0,0 @@
-<h1>Terms of Service</h1>
-
-<h3>Data Collected</h3>
-
-<p>Your username and a hash of your password, any posts you make and a list of accounts which you follow. The admin of the site does not know your password and it is not stored in plaintext anywhere.</p>
-
-<p>There is a quota on the number of posts retained by this instance for each account. Older posts will be removed when the limit is reached. Anything you post here should be considered ephemeral and you should keep a separate personal copy of them if you wish to retain a permanent archive.</p>
-
-<p>No IP addresses are logged.</p>
-
-<p>Posts can be removed on request if there is sufficient justification, but the nature of ActivityPub means that deletion of data federated to other instances cannot be guaranteed.</p>
-
-<h3>Content Policy</h3>
-
-<p>This instance will not host content containing sexism, racism, casteism, homophobia, transphobia, misogyny, antisemitism or other forms of bigotry or discrimination on the basis of nationality or immigration status. Claims that transgressions of this type were intended to be "ironic" will be treated as a terms of service violation.</p>
-
-<p>Even if not conspicuously discriminatory, expressions of support for organizations with discrminatory agendas are not permitted on this instance. These include, but are not limited to, racial supremacist groups, the redpill/incel movement and anti-LGBT or anti-immigrant campaigns.</p>
-
-<p>Depictions of injury, death or medical procedures are not permitted.</p>
-
-<p>Violent or abusive content will be subject to moderation and is likely to be removed.</p>
-
-<p>Content of a sexual nature may be published providing that only consenting adults (aged 18 or over) are depicted and an appropriate content warning message is added. Posting sexual content without a content warning is a terms of service violation. Sexual content is defined both as photographs of real people and also artistic or fictional depictions, edited/generated photos or narratives.</p>
-
-<p>Moderators rely upon your reports. Don't assume that something of concern has already been reported. It's better for there to be duplicate reports than for something potentially damaging to go unreported.</p>
-
-<p>Content found to be non-compliant with this policy will be removed and any accounts on this instance producing, repeating or linking to such content will be deleted typically without prior notification.</p>
-
-<h3>Federation Policy</h3>
-
-<p>In a proactive effort to avoid the classic fate of <i>"embrace, extend, extinguish"</i> this system will block any instance launched, acquired or funded by Alphabet, Facebook, Twitter, Microsoft, Apple, Amazon, Elsevier or other monopolistic Silicon Valley companies.</p>
-
-<p>This system will not federate with instances whose moderation policy is incompatible with the content policy described above. If an instance lacks a moderation policy, or refuses to enforce one, it will be assumed to be incompatible.</p>
-
-<h3>Use of User Generated Content for Research</h3>
-
-<p>Data may not be "scraped" or otherwise obtained from this instance and used for academic research or cited within research publications without the prior written permission of the administrator. Financial remedy will be sought through the courts from any researcher publishing data obtained from this instance without consent.</p>
-
-<h3>Commercial Use</h3>
-
-<p>Commercial use of original content on this instance is strictly forbidden without the prior written permission of individual account holders. The instance administrator does not hold copyright on any original content posted by account holders. Publication or federation of content does not imply permission for commercial use.</p>
-
-<p>Commercial use includes the harvesting of data to create products which are then sold, such as statistics, business reports or machine learning models.</p>
-
-<h3>Copyrights</h3>
-
-<p>Epicyon is licensed under <a href="https://www.gnu.org/licenses/agpl-3.0-standalone.html">GNU AGPL version 3</a>
-
-<p>Emojis designed by <a href="https://openmoji.org">OpenMoji</a> – the open-source emoji and icon project. License: <a href="https://creativecommons.org/licenses/by-sa/4.0">CC BY-SA 4.0</a></p>
-
-<p>Blob Cat Emoji and Meowmoji were made by Nitro Blob Hub, licensed under <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache 2.0</a>.</p>

epicyon.py

@@ -75,6 +75,10 @@ from theme import setTheme
 from announce import sendAnnounceViaServer
 from socnet import instancesGraph
 from migrate import migrateAccounts
+from speaker import getSpeakerFromServer
+from speaker import getSpeakerPitch
+from speaker import getSpeakerRate
+from speaker import getSpeakerRange
 import argparse
 
 
@@ -429,6 +433,10 @@ parser.add_argument('--level', dest='skillLevelPercent', type=int,
 parser.add_argument('--status', '--availability', dest='availability',
                     type=str, default=None,
                     help='Set an availability status')
+parser.add_argument('--speaker', '--tts', dest='speaker',
+                    type=str, default=None,
+                    help='Announce posts as they arrive at your ' +
+                    'inbox using TTS. --speaker [handle]')
 parser.add_argument('--block', dest='block', type=str, default=None,
                     help='Block a particular address')
 parser.add_argument('--unblock', dest='unblock', type=str, default=None,
@@ -1887,6 +1895,65 @@ if args.availability:
         time.sleep(1)
     sys.exit()
 
+if args.speaker:
+    # Announce posts as they arrive in your inbox using text-to-speech
+    if args.speaker.startswith('@'):
+        args.speaker = args.speaker[1:]
+    if '@' not in args.speaker:
+        print('Specify the handle of the speaker nickname@domain')
+        sys.exit()
+    nickname = args.speaker.split('@')[0]
+    domain = args.speaker.split('@')[1]
+
+    if not nickname:
+        print('Specify a nickname with the --nickname option')
+        sys.exit()
+
+    if not args.password:
+        print('Specify a password with the --password option')
+        sys.exit()
+
+    proxyType = None
+    if args.tor or domain.endswith('.onion'):
+        proxyType = 'tor'
+        if domain.endswith('.onion'):
+            args.port = 80
+    elif args.i2p or domain.endswith('.i2p'):
+        proxyType = 'i2p'
+        if domain.endswith('.i2p'):
+            args.port = 80
+    elif args.gnunet:
+        proxyType = 'gnunet'
+
+    print('Setting up espeak')
+    from espeak import espeak
+
+    session = createSession(proxyType)
+    print('Running speaker for ' + nickname + '@' + domain)
+
+    prevSay = ''
+    while (1):
+        speakerJson = \
+            getSpeakerFromServer(baseDir, session, nickname, args.password,
+                                 domain, port,
+                                 httpPrefix,
+                                 True, __version__)
+        if speakerJson:
+            if speakerJson['say'] != prevSay:
+                print(speakerJson['name'] + ': ' + speakerJson['say'] + '\n')
+                pitch = getSpeakerPitch(speakerJson['name'])
+                espeak.set_parameter(espeak.Parameter.Pitch, pitch)
+                rate = getSpeakerRate(speakerJson['name'])
+                espeak.set_parameter(espeak.Parameter.Rate, 110)
+                srange = getSpeakerRange(speakerJson['name'])
+                espeak.set_parameter(espeak.Parameter.Range, srange)
+                espeak.synth(speakerJson['name'])
+                time.sleep(3)
+                espeak.synth(speakerJson['say'])
+                prevSay = speakerJson['say']
+        time.sleep(20)
+    sys.exit()
+
 if federationList:
     print('Federating with: ' + str(federationList))
 

inbox.py

@@ -10,7 +10,10 @@ import json
 import os
 import datetime
 import time
+import urllib.parse
 from linked_data_sig import verifyJsonSignature
+from utils import getDisplayName
+from utils import removeHtml
 from utils import getConfigParam
 from utils import hasUsersPath
 from utils import validPostDate
@@ -77,6 +80,7 @@ from happening import saveEventPost
 from delete import removeOldHashtags
 from categories import guessHashtagCategory
 from context import hasValidContext
+from content import htmlReplaceQuoteMarks
 
 
 def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
@@ -2134,6 +2138,38 @@ def _bounceDM(senderPostId: str, session, httpPrefix: str,
     return True
 
 
+def _updateSpeaker(baseDir: str, nickname: str, domain: str,
+                   postJsonObject: {}, personCache: {}) -> None:
+    """ Generates a json file which can be used for TTS announcement
+    of incoming inbox posts
+    """
+    if not postJsonObject.get('object'):
+        return
+    if not isinstance(postJsonObject['object'], dict):
+        return
+    if not postJsonObject['object'].get('content'):
+        return
+    if not isinstance(postJsonObject['object']['content'], str):
+        return
+    speakerFilename = \
+        baseDir + '/accounts/' + nickname + '@' + domain + '/speaker.json'
+    content = urllib.parse.unquote_plus(postJsonObject['object']['content'])
+    content = removeHtml(htmlReplaceQuoteMarks(content))
+    summary = ''
+    if postJsonObject['object'].get('summary'):
+        if isinstance(postJsonObject['object']['summary'], str):
+            summary = \
+                urllib.parse.unquote_plus(postJsonObject['object']['summary'])
+    speakerName = \
+        getDisplayName(baseDir, postJsonObject['actor'], personCache)
+    speakerJson = {
+        "name": speakerName,
+        "summary": summary,
+        "say": content
+    }
+    saveJson(speakerJson, speakerFilename)
+
+
 def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                        session, keyId: str, handle: str, messageJson: {},
                        baseDir: str, httpPrefix: str, sendThreads: [],
@@ -2468,6 +2504,9 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                                         destinationFilename, debug):
                     print('ERROR: unable to update ' + boxname + ' index')
                 else:
+                    if boxname == 'inbox':
+                        _updateSpeaker(baseDir, nickname, domain,
+                                       postJsonObject, personCache)
                     if not unitTest:
                         if debug:
                             print('Saving inbox post as html to cache')

speaker.py

@@ -0,0 +1,65 @@
+__filename__ = "speaker.py"
+__author__ = "Bob Mottram"
+__license__ = "AGPL3+"
+__version__ = "1.2.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@freedombone.net"
+__status__ = "Production"
+
+import random
+from auth import createBasicAuthHeader
+from session import getJson
+from utils import getFullDomain
+
+
+def getSpeakerPitch(displayName: str) -> int:
+    """Returns the speech synthesis pitch for the given name
+    """
+    random.seed(displayName)
+    return random.randint(1, 100)
+
+
+def getSpeakerRate(displayName: str) -> int:
+    """Returns the speech synthesis rate for the given name
+    """
+    random.seed(displayName)
+    return random.randint(50, 120)
+
+
+def getSpeakerRange(displayName: str) -> int:
+    """Returns the speech synthesis range for the given name
+    """
+    random.seed(displayName)
+    return random.randint(300, 800)
+
+
+def getSpeakerFromServer(baseDir: str, session,
+                         nickname: str, password: str,
+                         domain: str, port: int,
+                         httpPrefix: str,
+                         debug: bool, projectVersion: str) -> {}:
+    """Returns some json which contains the latest inbox
+    entry in a minimal format suitable for a text-to-speech reader
+    """
+    if not session:
+        print('WARN: No session for getSpeakerFromServer')
+        return 6
+
+    domainFull = getFullDomain(domain, port)
+
+    authHeader = createBasicAuthHeader(nickname, password)
+
+    headers = {
+        'host': domain,
+        'Content-type': 'application/json',
+        'Authorization': authHeader
+    }
+
+    url = \
+        httpPrefix + '://' + \
+        domainFull + '/users/' + nickname + '/speaker'
+
+    speakerJson = \
+        getJson(session, url, headers, None,
+                __version__, httpPrefix, domain)
+    return speakerJson

theme/hacker/theme.json

@@ -1,4 +1,8 @@
 {
+    "dropdown-fg-color": "#dddddd",
+    "dropdown-bg-color": "#111",
+    "dropdown-bg-color-hover": "#035103",
+    "dropdown-fg-color-hover": "#dddddd",
     "newswire-publish-icon": "True",
     "full-width-timeline-buttons": "False",
     "icons-as-buttons": "False",

webapp_about.py

@@ -11,6 +11,7 @@ from shutil import copyfile
 from utils import getConfigParam
 from webapp_utils import htmlHeaderWithExternalStyle
 from webapp_utils import htmlFooter
+from webapp_utils import markdownToHtml
 
 
 def htmlAbout(cssCache: {}, baseDir: str, httpPrefix: str,
@@ -18,9 +19,9 @@ def htmlAbout(cssCache: {}, baseDir: str, httpPrefix: str,
     """Show the about screen
     """
     adminNickname = getConfigParam(baseDir, 'admin')
-    if not os.path.isfile(baseDir + '/accounts/about.txt'):
+    if not os.path.isfile(baseDir + '/accounts/about.md'):
-        copyfile(baseDir + '/default_about.txt',
+        copyfile(baseDir + '/default_about.md',
-                 baseDir + '/accounts/about.txt')
+                 baseDir + '/accounts/about.md')
 
     if os.path.isfile(baseDir + '/accounts/login-background-custom.jpg'):
         if not os.path.isfile(baseDir + '/accounts/login-background.jpg'):
@@ -28,9 +29,9 @@ def htmlAbout(cssCache: {}, baseDir: str, httpPrefix: str,
                      baseDir + '/accounts/login-background.jpg')
 
     aboutText = 'Information about this instance goes here.'
-    if os.path.isfile(baseDir + '/accounts/about.txt'):
+    if os.path.isfile(baseDir + '/accounts/about.md'):
-        with open(baseDir + '/accounts/about.txt', 'r') as aboutFile:
+        with open(baseDir + '/accounts/about.md', 'r') as aboutFile:
-            aboutText = aboutFile.read()
+            aboutText = markdownToHtml(aboutFile.read())
 
     aboutForm = ''
     cssFilename = baseDir + '/epicyon-profile.css'

webapp_column_left.py

@@ -411,7 +411,7 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str,
     adminNickname = getConfigParam(baseDir, 'admin')
     if adminNickname:
         if nickname == adminNickname:
-            aboutFilename = baseDir + '/accounts/about.txt'
+            aboutFilename = baseDir + '/accounts/about.md'
             aboutStr = ''
             if os.path.isfile(aboutFilename):
                 with open(aboutFilename, 'r') as fp:
@@ -430,7 +430,7 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str,
             editLinksForm += \
                 '</div>'
 
-            TOSFilename = baseDir + '/accounts/tos.txt'
+            TOSFilename = baseDir + '/accounts/tos.md'
             TOSStr = ''
             if os.path.isfile(TOSFilename):
                 with open(TOSFilename, 'r') as fp:

webapp_tos.py

@@ -11,6 +11,7 @@ from shutil import copyfile
 from utils import getConfigParam
 from webapp_utils import htmlHeaderWithExternalStyle
 from webapp_utils import htmlFooter
+from webapp_utils import markdownToHtml
 
 
 def htmlTermsOfService(cssCache: {}, baseDir: str,
@@ -18,9 +19,9 @@ def htmlTermsOfService(cssCache: {}, baseDir: str,
     """Show the terms of service screen
     """
     adminNickname = getConfigParam(baseDir, 'admin')
-    if not os.path.isfile(baseDir + '/accounts/tos.txt'):
+    if not os.path.isfile(baseDir + '/accounts/tos.md'):
-        copyfile(baseDir + '/default_tos.txt',
+        copyfile(baseDir + '/default_tos.md',
-                 baseDir + '/accounts/tos.txt')
+                 baseDir + '/accounts/tos.md')
 
     if os.path.isfile(baseDir + '/accounts/login-background-custom.jpg'):
         if not os.path.isfile(baseDir + '/accounts/login-background.jpg'):
@@ -28,9 +29,9 @@ def htmlTermsOfService(cssCache: {}, baseDir: str,
                      baseDir + '/accounts/login-background.jpg')
 
     TOSText = 'Terms of Service go here.'
-    if os.path.isfile(baseDir + '/accounts/tos.txt'):
+    if os.path.isfile(baseDir + '/accounts/tos.md'):
-        with open(baseDir + '/accounts/tos.txt', 'r') as file:
+        with open(baseDir + '/accounts/tos.md', 'r') as file:
-            TOSText = file.read()
+            TOSText = markdownToHtml(file.read())
 
     TOSForm = ''
     cssFilename = baseDir + '/epicyon-profile.css'