Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon

newsdaemon.py

@@ -74,22 +74,26 @@ def _removeControlCharacters(content: str) -> str:
     return content
 
 
-def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
+def _hashtagLogicalNot(tree: [], hashtags: [], moderated: bool,
                        content: str, url: str) -> bool:
-    """Returns whether the tree for a hashtag rule evaluates to true or false
+    """ NOT
     """
-    if not tree:
+    if len(tree) != 2:
         return False
-
-    if tree[0] == 'not':
-        if len(tree) == 2:
     if isinstance(tree[1], str):
         return tree[1] not in hashtags
     elif isinstance(tree[1], list):
-                return not hashtagRuleResolve(tree[1], hashtags, moderated,
+        return not hashtagRuleResolve(tree[1], hashtags,
-                                              content, url)
+                                      moderated, content, url)
-    elif tree[0] == 'contains':
+    return False
-        if len(tree) == 2:
+
+
+def _hashtagLogicalContains(tree: [], hashtags: [], moderated: bool,
+                            content: str, url: str) -> bool:
+    """ Contains
+    """
+    if len(tree) != 2:
+        return False
     matchStr = None
     if isinstance(tree[1], str):
         matchStr = tree[1]
@@ -102,8 +106,15 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
         matchStrLower = matchStr.lower()
         contentWithoutTags = content.replace('#' + matchStrLower, '')
         return matchStrLower in contentWithoutTags
-    elif tree[0] == 'from':
+    return False
-        if len(tree) == 2:
+
+
+def _hashtagLogicalFrom(tree: [], hashtags: [], moderated: bool,
+                        content: str, url: str) -> bool:
+    """ FROM
+    """
+    if len(tree) != 2:
+        return False
     matchStr = None
     if isinstance(tree[1], str):
         matchStr = tree[1]
@@ -114,8 +125,15 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
             matchStr = matchStr[1:]
             matchStr = matchStr[:len(matchStr) - 1]
         return matchStr.lower() in url
-    elif tree[0] == 'and':
+    return False
-        if len(tree) >= 3:
+
+
+def _hashtagLogicalAnd(tree: [], hashtags: [], moderated: bool,
+                       content: str, url: str) -> bool:
+    """ AND
+    """
+    if len(tree) < 3:
+        return False
     for argIndex in range(1, len(tree)):
         argValue = False
         if isinstance(tree[argIndex], str):
@@ -127,8 +145,14 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
         if not argValue:
             return False
     return True
-    elif tree[0] == 'or':
+
-        if len(tree) >= 3:
+
+def _hashtagLogicalOr(tree: [], hashtags: [], moderated: bool,
+                      content: str, url: str) -> bool:
+    """ OR
+    """
+    if len(tree) < 3:
+        return False
     for argIndex in range(1, len(tree)):
         argValue = False
         if isinstance(tree[argIndex], str):
@@ -140,8 +164,14 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
         if argValue:
             return True
     return False
-    elif tree[0] == 'xor':
+
-        if len(tree) >= 3:
+
+def _hashtagLogicalXor(tree: [], hashtags: [], moderated: bool,
+                       content: str, url: str) -> bool:
+    """ XOR
+    """
+    if len(tree) < 3:
+        return False
     trueCtr = 0
     for argIndex in range(1, len(tree)):
         argValue = False
@@ -155,6 +185,28 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
             trueCtr += 1
     if trueCtr == 1:
         return True
+    return False
+
+
+def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
+                       content: str, url: str) -> bool:
+    """Returns whether the tree for a hashtag rule evaluates to true or false
+    """
+    if not tree:
+        return False
+
+    if tree[0] == 'not':
+        return _hashtagLogicalNot(tree, hashtags, moderated, content, url)
+    elif tree[0] == 'contains':
+        return _hashtagLogicalContains(tree, hashtags, moderated, content, url)
+    elif tree[0] == 'from':
+        return _hashtagLogicalFrom(tree, hashtags, moderated, content, url)
+    elif tree[0] == 'and':
+        return _hashtagLogicalAnd(tree, hashtags, moderated, content, url)
+    elif tree[0] == 'or':
+        return _hashtagLogicalOr(tree, hashtags, moderated, content, url)
+    elif tree[0] == 'xor':
+        return _hashtagLogicalXor(tree, hashtags, moderated, content, url)
     elif tree[0].startswith('#') and len(tree) == 1:
         return tree[0] in hashtags
     elif tree[0].startswith('moderated'):
@@ -225,6 +277,87 @@ def hashtagRuleTree(operators: [],
     return tree
 
 
+def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
+                postJsonObject: {},
+                actionStr: str, hashtags: []) -> None:
+    """Adds a hashtag via a hashtag rule
+    """
+    addHashtag = actionStr.split('add ', 1)[1].strip()
+    if not addHashtag.startswith('#'):
+        return
+
+    if addHashtag not in hashtags:
+        hashtags.append(addHashtag)
+    htId = addHashtag.replace('#', '')
+    if not validHashTag(htId):
+        return
+
+    hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
+    newTag = {
+        'href': hashtagUrl,
+        'name': addHashtag,
+        'type': 'Hashtag'
+    }
+    # does the tag already exist?
+    addTagObject = None
+    for t in postJsonObject['object']['tag']:
+        if t.get('type') and t.get('name'):
+            if t['type'] == 'Hashtag' and \
+               t['name'] == addHashtag:
+                addTagObject = t
+                break
+    # append the tag if it wasn't found
+    if not addTagObject:
+        postJsonObject['object']['tag'].append(newTag)
+    # add corresponding html to the post content
+    hashtagHtml = \
+        " <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
+        "rel=\"tag\">#<span>" + htId + "</span></a>"
+    content = postJsonObject['object']['content']
+    if hashtagHtml in content:
+        return
+
+    if content.endswith('</p>'):
+        content = \
+            content[:len(content) - len('</p>')] + \
+            hashtagHtml + '</p>'
+    else:
+        content += hashtagHtml
+    postJsonObject['object']['content'] = content
+    storeHashTags(baseDir, 'news', postJsonObject)
+
+
+def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
+                   actionStr: str, hashtags: []) -> None:
+    """Removes a hashtag via a hashtag rule
+    """
+    rmHashtag = actionStr.split('remove ', 1)[1].strip()
+    if not rmHashtag.startswith('#'):
+        return
+
+    if rmHashtag in hashtags:
+        hashtags.remove(rmHashtag)
+    htId = rmHashtag.replace('#', '')
+    hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
+    # remove tag html from the post content
+    hashtagHtml = \
+        "<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
+        "rel=\"tag\">#<span>" + htId + "</span></a>"
+    content = postJsonObject['object']['content']
+    if hashtagHtml in content:
+        content = content.replace(hashtagHtml, '').replace('  ', ' ')
+        postJsonObject['object']['content'] = content
+    rmTagObject = None
+    for t in postJsonObject['object']['tag']:
+        if t.get('type') and t.get('name'):
+            if t['type'] == 'Hashtag' and \
+               t['name'] == rmHashtag:
+                rmTagObject = t
+                break
+    if rmTagObject:
+        postJsonObject['object']['tag'].remove(rmTagObject)
+
+
 def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
                                hashtags: [], httpPrefix: str,
                                domain: str, port: int,
@@ -273,83 +406,16 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
         # the condition matches, so do something
         actionStr = ruleStr.split(' then ')[1].strip()
 
-        # add a hashtag
         if actionStr.startswith('add '):
-            addHashtag = actionStr.split('add ', 1)[1].strip()
+            # add a hashtag
-            if addHashtag.startswith('#'):
+            _hashtagAdd(baseDir, httpPrefix, domainFull,
-                if addHashtag not in hashtags:
+                        postJsonObject, actionStr, hashtags)
-                    hashtags.append(addHashtag)
+        elif actionStr.startswith('remove '):
-                htId = addHashtag.replace('#', '')
-                if validHashTag(htId):
-                    hashtagUrl = \
-                        httpPrefix + "://" + domainFull + "/tags/" + htId
-                    newTag = {
-                        'href': hashtagUrl,
-                        'name': addHashtag,
-                        'type': 'Hashtag'
-                    }
-                    # does the tag already exist?
-                    addTagObject = None
-                    for t in postJsonObject['object']['tag']:
-                        if t.get('type') and t.get('name'):
-                            if t['type'] == 'Hashtag' and \
-                               t['name'] == addHashtag:
-                                addTagObject = t
-                                break
-                    # append the tag if it wasn't found
-                    if not addTagObject:
-                        postJsonObject['object']['tag'].append(newTag)
-                    # add corresponding html to the post content
-                    hashtagHtml = \
-                        " <a href=\"" + hashtagUrl + \
-                        "\" class=\"addedHashtag\" " + \
-                        "rel=\"tag\">#<span>" + \
-                        htId + "</span></a>"
-                    content = postJsonObject['object']['content']
-                    if hashtagHtml not in content:
-                        if content.endswith('</p>'):
-                            content = \
-                                content[:len(content) - len('</p>')] + \
-                                hashtagHtml + '</p>'
-                        else:
-                            content += hashtagHtml
-                        postJsonObject['object']['content'] = content
-                        storeHashTags(baseDir, 'news', postJsonObject)
-                        # actionOccurred = True
-
             # remove a hashtag
-        if actionStr.startswith('remove '):
+            _hashtagRemove(httpPrefix, domainFull, postJsonObject,
-            rmHashtag = actionStr.split('remove ', 1)[1].strip()
+                           actionStr, hashtags)
-            if rmHashtag.startswith('#'):
+        elif actionStr.startswith('block') or actionStr.startswith('drop'):
-                if rmHashtag in hashtags:
-                    hashtags.remove(rmHashtag)
-                htId = rmHashtag.replace('#', '')
-                hashtagUrl = \
-                    httpPrefix + "://" + domainFull + "/tags/" + htId
-                # remove tag html from the post content
-                hashtagHtml = \
-                    "<a href=\"" + hashtagUrl + \
-                    "\" class=\"addedHashtag\" " + \
-                    "rel=\"tag\">#<span>" + \
-                    htId + "</span></a>"
-                content = postJsonObject['object']['content']
-                if hashtagHtml in content:
-                    content = \
-                        content.replace(hashtagHtml, '').replace('  ', ' ')
-                    postJsonObject['object']['content'] = content
-                rmTagObject = None
-                for t in postJsonObject['object']['tag']:
-                    if t.get('type') and t.get('name'):
-                        if t['type'] == 'Hashtag' and \
-                           t['name'] == rmHashtag:
-                            rmTagObject = t
-                            break
-                if rmTagObject:
-                    postJsonObject['object']['tag'].remove(rmTagObject)
-                    # actionOccurred = True
-
             # Block this item
-        if actionStr.startswith('block') or actionStr.startswith('drop'):
             return False
     return True
 

newswire.py

@@ -47,21 +47,25 @@ def rss2Header(httpPrefix: str,
                title: str, translate: {}) -> str:
     """Header for an RSS 2.0 feed
     """
-    rssStr = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
+    rssStr = \
-    rssStr += "<rss version=\"2.0\">"
+        "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + \
-    rssStr += '<channel>'
+        "<rss version=\"2.0\">" + \
+        '<channel>'
 
     if title.startswith('News'):
-        rssStr += '    <title>Newswire</title>'
+        rssStr += \
-        rssStr += '    <link>' + httpPrefix + '://' + domainFull + \
+            '    <title>Newswire</title>' + \
+            '    <link>' + httpPrefix + '://' + domainFull + \
             '/newswire.xml' + '</link>'
     elif title.startswith('Site'):
-        rssStr += '    <title>' + domainFull + '</title>'
+        rssStr += \
-        rssStr += '    <link>' + httpPrefix + '://' + domainFull + \
+            '    <title>' + domainFull + '</title>' + \
+            '    <link>' + httpPrefix + '://' + domainFull + \
             '/blog/rss.xml' + '</link>'
     else:
-        rssStr += '    <title>' + translate[title] + '</title>'
+        rssStr += \
-        rssStr += '    <link>' + httpPrefix + '://' + domainFull + \
+            '    <title>' + translate[title] + '</title>' + \
+            '    <link>' + httpPrefix + '://' + domainFull + \
             '/users/' + nickname + '/rss.xml' + '</link>'
     return rssStr
 
@@ -69,8 +73,7 @@ def rss2Header(httpPrefix: str,
 def rss2Footer() -> str:
     """Footer for an RSS 2.0 feed
     """
-    rssStr = '</channel>'
+    rssStr = '</channel></rss>'
-    rssStr += '</rss>'
     return rssStr
 
 
@@ -815,8 +818,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
         except Exception as e:
             print('WARN: Unable to convert date ' + published + ' ' + str(e))
             continue
-        rssStr += '<item>\n'
+        rssStr += \
-        rssStr += '  <title>' + fields[0] + '</title>\n'
+            '<item>\n' + \
+            '  <title>' + fields[0] + '</title>\n'
         description = removeHtml(firstParagraphFromString(fields[4]))
         rssStr += '  <description>' + description + '</description>\n'
         url = fields[1]
@@ -826,8 +830,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
         rssStr += '  <link>' + url + '</link>\n'
 
         rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
-        rssStr += '  <pubDate>' + rssDateStr + '</pubDate>\n'
+        rssStr += \
-        rssStr += '</item>\n'
+            '  <pubDate>' + rssDateStr + '</pubDate>\n' + \
+            '</item>\n'
     rssStr += rss2Footer()
     return rssStr
 

outbox.py

@@ -311,8 +311,7 @@ def postMessageToOutbox(session, translate: {},
 
     permittedOutboxTypes = ('Create', 'Announce', 'Like', 'Follow', 'Undo',
                             'Update', 'Add', 'Remove', 'Block', 'Delete',
-                            'Skill', 'Add', 'Remove', 'Event',
+                            'Skill', 'Ignore')
-                            'Ignore')
     if messageJson['type'] not in permittedOutboxTypes:
         if debug:
             print('DEBUG: POST to outbox - ' + messageJson['type'] +
@@ -369,14 +368,12 @@ def postMessageToOutbox(session, translate: {},
                 if os.path.isfile(citationsFilename):
                     os.remove(citationsFilename)
 
-        if messageJson['type'] == 'Create' or \
+        # The following activity types get added to the index files
-           messageJson['type'] == 'Question' or \
+        indexedActivities = (
-           messageJson['type'] == 'Note' or \
+            'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
-           messageJson['type'] == 'EncryptedMessage' or \
+            'Patch', 'Announce'
-           messageJson['type'] == 'Article' or \
+        )
-           messageJson['type'] == 'Event' or \
+        if messageJson['type'] in indexedActivities:
-           messageJson['type'] == 'Patch' or \
-           messageJson['type'] == 'Announce':
             indexes = [outboxName, "inbox"]
             selfActor = \
                 httpPrefix + '://' + domainFull + '/users/' + postToNickname

person.py

@@ -49,6 +49,7 @@ from utils import refreshNewswire
 from utils import getProtocolPrefixes
 from utils import hasUsersPath
 from utils import getImageExtensions
+from utils import isImageFile
 from session import createSession
 from session import getJson
 from webfinger import webfingerHandle
@@ -84,11 +85,7 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
     image for the given person
     """
     imageFilename = imageFilename.replace('\n', '').replace('\r', '')
-    if not (imageFilename.endswith('.png') or
+    if not isImageFile(imageFilename):
-            imageFilename.endswith('.jpg') or
-            imageFilename.endswith('.jpeg') or
-            imageFilename.endswith('.svg') or
-            imageFilename.endswith('.gif')):
         print('Profile image must be png, jpg, gif or svg format')
         return False
 
@@ -119,10 +116,16 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
        imageFilename.endswith('.jpeg'):
         mediaType = 'image/jpeg'
         iconFilename = iconFilenameBase + '.jpg'
-    if imageFilename.endswith('.gif'):
+    elif imageFilename.endswith('.gif'):
         mediaType = 'image/gif'
         iconFilename = iconFilenameBase + '.gif'
-    if imageFilename.endswith('.svg'):
+    elif imageFilename.endswith('.webp'):
+        mediaType = 'image/webp'
+        iconFilename = iconFilenameBase + '.webp'
+    elif imageFilename.endswith('.avif'):
+        mediaType = 'image/avif'
+        iconFilename = iconFilenameBase + '.avif'
+    elif imageFilename.endswith('.svg'):
         mediaType = 'image/svg+xml'
         iconFilename = iconFilenameBase + '.svg'
     profileFilename = baseDir + '/accounts/' + handle + '/' + iconFilename
@@ -593,8 +596,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
     # if the older skills format is being used then switch
     # to the new one
     if not personJson.get('hasOccupation'):
-        personJson['hasOccupation'] = [
+        personJson['hasOccupation'] = [{
-            {
             '@type': 'Occupation',
             'name': occupationName,
             "occupationLocation": {
@@ -602,8 +604,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
                 "name": "Fediverse"
             },
             'skills': []
-            }
+        }]
-        ]
         updateActor = True
 
     # remove the old skills format
@@ -618,8 +619,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
         updateActor = True
 
     if not isinstance(personJson['hasOccupation'], list):
-        personJson['hasOccupation'] = [
+        personJson['hasOccupation'] = [{
-            {
             '@type': 'Occupation',
             'name': occupationName,
             'occupationLocation': {
@@ -627,8 +627,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
                 'name': 'Fediverse'
             },
             'skills': []
-            }
+        }]
-        ]
         updateActor = True
     else:
         # add location if it is missing
@@ -1212,27 +1211,17 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool,
         for prefix in prefixes:
             handle = handle.replace(prefix, '')
         handle = handle.replace('/@', '/users/')
-        if '/users/' in handle:
+        paths = (
-            nickname = handle.split('/users/')[1]
+            '/users/', '/profile/', '/channel/', '/accounts/', '/u/'
+        )
+        userPathFound = False
+        for userPath in paths:
+            if userPath in handle:
+                nickname = handle.split(userPath)[1]
                 nickname = nickname.replace('\n', '').replace('\r', '')
-            domain = handle.split('/users/')[0]
+                domain = handle.split(userPath)[0]
-        elif '/profile/' in handle:
+                break
-            nickname = handle.split('/profile/')[1]
+        if not userPathFound and '://' in originalHandle:
-            nickname = nickname.replace('\n', '').replace('\r', '')
-            domain = handle.split('/profile/')[0]
-        elif '/channel/' in handle:
-            nickname = handle.split('/channel/')[1]
-            nickname = nickname.replace('\n', '').replace('\r', '')
-            domain = handle.split('/channel/')[0]
-        elif '/accounts/' in handle:
-            nickname = handle.split('/accounts/')[1]
-            nickname = nickname.replace('\n', '').replace('\r', '')
-            domain = handle.split('/accounts/')[0]
-        elif '/u/' in handle:
-            nickname = handle.split('/u/')[1]
-            nickname = nickname.replace('\n', '').replace('\r', '')
-            domain = handle.split('/u/')[0]
-        elif '://' in originalHandle:
             domain = originalHandle.split('://')[1]
             if '/' in domain:
                 domain = domain.split('/')[0]

posts.py

@@ -492,9 +492,9 @@ def _updateWordFrequency(content: str, wordFrequency: {}) -> None:
     that they appear
     """
     plainText = removeHtml(content)
-    plainText = plainText.replace('.', ' ')
+    removeChars = ('.', ';', '?')
-    plainText = plainText.replace(';', ' ')
+    for ch in removeChars:
-    plainText = plainText.replace('?', ' ')
+        plainText = plainText.replace(ch, ' ')
     wordsList = plainText.split(' ')
     commonWords = (
         'that', 'some', 'about', 'then', 'they', 'were',
@@ -1656,7 +1656,8 @@ def getMentionedPeople(baseDir: str, httpPrefix: str,
     mentions = []
     words = content.split(' ')
     for wrd in words:
-        if wrd.startswith('@'):
+        if not wrd.startswith('@'):
+            continue
         handle = wrd[1:]
         if debug:
             print('DEBUG: mentioned handle ' + handle)
@@ -2140,7 +2141,8 @@ def groupFollowersByDomain(baseDir: str, nickname: str, domain: str) -> {}:
     grouped = {}
     with open(followersFilename, "r") as f:
         for followerHandle in f:
-            if '@' in followerHandle:
+            if '@' not in followerHandle:
+                continue
             fHandle = \
                 followerHandle.strip().replace('\n', '').replace('\r', '')
             followerDomain = fHandle.split('@')[1]
@@ -2174,9 +2176,9 @@ def _addFollowersToPublicPost(postJsonObject: {}) -> None:
             return
         if len(postJsonObject['object']['to']) > 1:
             return
-        if len(postJsonObject['object']['to']) == 0:
+        elif len(postJsonObject['object']['to']) == 0:
             return
-        if not postJsonObject['object']['to'][0].endswith('#Public'):
+        elif not postJsonObject['object']['to'][0].endswith('#Public'):
             return
         if postJsonObject['object'].get('cc'):
             return
@@ -2403,6 +2405,20 @@ def addToField(activityType: str, postJsonObject: {},
     return postJsonObject, False
 
 
+def _isProfileUpdate(postJsonObject: {}) -> bool:
+    """Is the given post a profile update?
+    for actor updates there is no 'to' within the object
+    """
+    if postJsonObject['object'].get('type') and postJsonObject.get('type'):
+        if (postJsonObject['type'] == 'Update' and
+            (postJsonObject['object']['type'] == 'Person' or
+             postJsonObject['object']['type'] == 'Application' or
+             postJsonObject['object']['type'] == 'Group' or
+             postJsonObject['object']['type'] == 'Service')):
+            return True
+    return False
+
+
 def sendToNamedAddresses(session, baseDir: str,
                          nickname: str, domain: str,
                          onionDomain: str, i2pDomain: str, port: int,
@@ -2420,13 +2436,7 @@ def sendToNamedAddresses(session, baseDir: str,
         return
     isProfileUpdate = False
     if isinstance(postJsonObject['object'], dict):
-        # for actor updates there is no 'to' within the object
+        if _isProfileUpdate(postJsonObject):
-        if postJsonObject['object'].get('type') and postJsonObject.get('type'):
-            if (postJsonObject['type'] == 'Update' and
-                (postJsonObject['object']['type'] == 'Person' or
-                 postJsonObject['object']['type'] == 'Application' or
-                 postJsonObject['object']['type'] == 'Group' or
-                 postJsonObject['object']['type'] == 'Service')):
             # use the original object, which has a 'to'
             recipientsObject = postJsonObject
             isProfileUpdate = True