Merge branches 'main' and 'main' of gitlab.com:bashrc2/epicyon

blog.py

@@ -164,7 +164,7 @@ def _getBlogReplies(baseDir: str, httpPrefix: str, translate: {},
     return ''
 
 
-def _htmlBlogPostContent(authorized: bool,
+def _htmlBlogPostContent(debug: bool, session, authorized: bool,
                          baseDir: str, httpPrefix: str, translate: {},
                          nickname: str, domain: str, domainFull: str,
                          postJsonObject: {},
@@ -255,9 +255,9 @@ def _htmlBlogPostContent(authorized: bool,
         contentStr = addEmbeddedElements(translate, jsonContent,
                                          peertubeInstances)
         if postJsonObject['object'].get('tag'):
-            contentStr = replaceEmojiFromTags(contentStr,
+            contentStr = replaceEmojiFromTags(session, baseDir, contentStr,
                                               postJsonObject['object']['tag'],
-                                              'content')
+                                              'content', debug)
         if articleAdded:
             blogStr += '<br>' + contentStr + '</article>\n'
         else:
@@ -414,12 +414,13 @@ def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str:
     return content
 
 
-def htmlBlogPost(authorized: bool,
+def htmlBlogPost(session, authorized: bool,
                  baseDir: str, httpPrefix: str, translate: {},
                  nickname: str, domain: str, domainFull: str,
                  postJsonObject: {},
                  peertubeInstances: [],
-                 systemLanguage: str, personCache: {}) -> str:
+                 systemLanguage: str, personCache: {},
+                 debug: bool) -> str:
     """Returns a html blog post
     """
     blogStr = ''
@@ -438,7 +439,7 @@ def htmlBlogPost(authorized: bool,
                                        title, snippet)
     _htmlBlogRemoveCwButton(blogStr, translate)
 
-    blogStr += _htmlBlogPostContent(authorized, baseDir,
+    blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir,
                                     httpPrefix, translate,
                                     nickname, domain,
                                     domainFull, postJsonObject,
@@ -473,7 +474,7 @@ def htmlBlogPage(authorized: bool, session,
                  nickname: str, domain: str, port: int,
                  noOfItems: int, pageNumber: int,
                  peertubeInstances: [], systemLanguage: str,
-                 personCache: {}) -> str:
+                 personCache: {}, debug: bool) -> str:
     """Returns a html blog page containing posts
     """
     if ' ' in nickname or '@' in nickname or \
@@ -530,7 +531,7 @@ def htmlBlogPage(authorized: bool, session,
         if item['type'] != 'Create':
             continue
 
-        blogStr += _htmlBlogPostContent(authorized, baseDir,
+        blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir,
                                         httpPrefix, translate,
                                         nickname, domain,
                                         domainFull, item,
@@ -696,7 +697,7 @@ def htmlBlogView(authorized: bool,
                  translate: {}, domain: str, port: int,
                  noOfItems: int,
                  peertubeInstances: [], systemLanguage: str,
-                 personCache: {}) -> str:
+                 personCache: {}, debug: bool) -> str:
     """Show the blog main page
     """
     blogStr = ''
@@ -715,7 +716,7 @@ def htmlBlogView(authorized: bool,
                                 baseDir, httpPrefix, translate,
                                 nickname, domain, port,
                                 noOfItems, 1, peertubeInstances,
-                                systemLanguage, personCache)
+                                systemLanguage, personCache, debug)
 
     domainFull = getFullDomain(domain, port)
 

content.py

@@ -16,6 +16,7 @@ from utils import removeDomainPort
 from utils import isValidLanguage
 from utils import getImageExtensions
 from utils import loadJson
+from utils import saveJson
 from utils import fileLastModified
 from utils import getLinkPrefixes
 from utils import dangerousMarkup
@@ -26,6 +27,7 @@ from utils import isfloat
 from utils import getCurrencies
 from utils import removeHtml
 from petnames import getPetName
+from session import downloadImage
 
 
 def removeHtmlTag(htmlStr: str, tag: str) -> str:
@@ -239,7 +241,49 @@ def switchWords(baseDir: str, nickname: str, domain: str, content: str,
     return content
 
 
-def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
+def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str,
+                     debug: bool) -> None:
+    """Saves custom emoji to file
+    """
+    if not session:
+        if debug:
+            print('EX: _saveCustomEmoji no session')
+        return
+    if '.' not in url:
+        return
+    ext = url.split('.')[-1]
+    if ext != 'png':
+        if debug:
+            print('EX: Custom emoji is wrong format ' + url)
+        return
+    emojiName = emojiName.replace(':', '').strip().lower()
+    customEmojiDir = baseDir + '/emojicustom'
+    if not os.path.isdir(customEmojiDir):
+        os.mkdir(customEmojiDir)
+    emojiImageFilename = customEmojiDir + '/' + emojiName + '.' + ext
+    if not downloadImage(session, baseDir, url,
+                         emojiImageFilename, debug, False):
+        if debug:
+            print('EX: custom emoji not downloaded ' + url)
+        return
+    emojiJsonFilename = customEmojiDir + '/emoji.json'
+    emojiJson = {}
+    if os.path.isfile(emojiJsonFilename):
+        emojiJson = loadJson(emojiJsonFilename, 0, 1)
+        if not emojiJson:
+            emojiJson = {}
+    if not emojiJson.get(emojiName):
+        emojiJson[emojiName] = emojiName
+        saveJson(emojiJson, emojiJsonFilename)
+        if debug:
+            print('EX: Saved custom emoji ' + emojiJsonFilename)
+    elif debug:
+        print('EX: cusom emoji already saved')
+
+
+def replaceEmojiFromTags(session, baseDir: str,
+                         content: str, tag: [], messageType: str,
+                         debug: bool) -> str:
     """Uses the tags to replace :emoji: with html image markup
     """
     for tagItem in tag:
@@ -267,33 +311,45 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
                         # emoji/charts/full-emoji-list.html
                         if '-' not in iconName:
                             # a single code
+                            replaced = False
                             try:
                                 replaceChar = chr(int("0x" + iconName, 16))
                                 content = content.replace(tagItem['name'],
                                                           replaceChar)
+                                replaced = True
                             except BaseException:
-                                print('EX: replaceEmojiFromTags ' +
+                                print('EX: replaceEmojiFromTags 1 ' +
                                       'no conversion of ' +
                                       str(iconName) + ' to chr ' +
                                       tagItem['name'] + ' ' +
                                       tagItem['icon']['url'])
-                                pass
+                            if not replaced:
+                                _saveCustomEmoji(session, baseDir,
+                                                 tagItem['name'],
+                                                 tagItem['icon']['url'],
+                                                 debug)
                         else:
                             # sequence of codes
                             iconCodes = iconName.split('-')
                             iconCodeSequence = ''
                             for icode in iconCodes:
+                                replaced = False
                                 try:
                                     iconCodeSequence += chr(int("0x" +
                                                                 icode, 16))
+                                    replaced = True
                                 except BaseException:
                                     iconCodeSequence = ''
-                                    print('EX: replaceEmojiFromTags ' +
+                                    print('EX: replaceEmojiFromTags 2 ' +
                                           'no conversion of ' +
                                           str(icode) + ' to chr ' +
                                           tagItem['name'] + ' ' +
                                           tagItem['icon']['url'])
-                                    break
+                                if not replaced:
+                                    _saveCustomEmoji(session, baseDir,
+                                                     tagItem['name'],
+                                                     tagItem['icon']['url'],
+                                                     debug)
                             if iconCodeSequence:
                                 content = content.replace(tagItem['name'],
                                                           iconCodeSequence)
@@ -844,6 +900,13 @@ def addHtmlTags(baseDir: str, httpPrefix: str,
                                  baseDir + '/emoji/emoji.json')
                 emojiDict = loadJson(baseDir + '/emoji/emoji.json')
 
+                # append custom emoji to the dict
+                if os.path.isfile(baseDir + '/emojicustom/emoji.json'):
+                    customEmojiDict = \
+                        loadJson(baseDir + '/emojicustom/emoji.json')
+                    if customEmojiDict:
+                        emojiDict = dict(emojiDict, **customEmojiDict)
+
 #                print('TAG: looking up emoji for :' + wordStr2 + ':')
                 _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix,
                           originalDomain, replaceEmoji, hashtags,

daemon.py

@@ -6723,6 +6723,8 @@ class PubServer(BaseHTTPRequestHandler):
         if isImageFile(path):
             emojiStr = path.split('/emoji/')[1]
             emojiFilename = baseDir + '/emoji/' + emojiStr
+            if not os.path.isfile(emojiFilename):
+                emojiFilename = baseDir + '/emojicustom/' + emojiStr
             if os.path.isfile(emojiFilename):
                 if self._etag_exists(emojiFilename):
                     # The file has not changed
@@ -11379,7 +11381,8 @@ class PubServer(BaseHTTPRequestHandler):
                            maxPostsInBlogsFeed, pageNumber,
                            self.server.peertubeInstances,
                            self.server.systemLanguage,
-                           self.server.personCache)
+                           self.server.personCache,
+                           self.server.debug)
         if msg is not None:
             msg = msg.encode('utf-8')
             msglen = len(msg)
@@ -12983,7 +12986,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    maxPostsInBlogsFeed,
                                    self.server.peertubeInstances,
                                    self.server.systemLanguage,
-                                   self.server.personCache)
+                                   self.server.personCache,
+                                   self.server.debug)
                 if msg is not None:
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
@@ -13076,7 +13080,8 @@ class PubServer(BaseHTTPRequestHandler):
             if blogFilename and nickname:
                 postJsonObject = loadJson(blogFilename)
                 if isBlogPost(postJsonObject):
-                    msg = htmlBlogPost(authorized,
+                    msg = htmlBlogPost(self.server.session,
+                                       authorized,
                                        self.server.baseDir,
                                        self.server.httpPrefix,
                                        self.server.translate,
@@ -13085,7 +13090,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        postJsonObject,
                                        self.server.peertubeInstances,
                                        self.server.systemLanguage,
-                                       self.server.personCache)
+                                       self.server.personCache,
+                                       self.server.debug)
                     if msg is not None:
                         msg = msg.encode('utf-8')
                         msglen = len(msg)
@@ -15538,8 +15544,11 @@ class PubServer(BaseHTTPRequestHandler):
                             tags.append(tag)
                         # get list of tags
                         fields['message'] = \
-                            replaceEmojiFromTags(fields['message'],
+                            replaceEmojiFromTags(self.server.session,
-                                                 tags, 'content')
+                                                 self.server.baseDir,
+                                                 fields['message'],
+                                                 tags, 'content',
+                                                 self.server.debug)
 
                         postJsonObject['object']['content'] = fields['message']
                         contentMap = postJsonObject['object']['contentMap']

posts.py

@@ -1292,7 +1292,8 @@ def _createPostModReport(baseDir: str,
         modFile.write(newPostId + '\n')
 
 
-def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
+def _createPostBase(baseDir: str,
+                    nickname: str, domain: str, port: int,
                     toUrl: str, ccUrl: str, httpPrefix: str, content: str,
                     followersOnly: bool, saveToFile: bool,
                     clientToServer: bool, commentsEnabled: bool,
@@ -1343,9 +1344,12 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
     tags = []
     for tagName, tag in hashtagsDict.items():
         tags.append(tag)
+
     # get list of tags
     if nickname != 'news':
-        content = replaceEmojiFromTags(content, tags, 'content')
+        content = \
+            replaceEmojiFromTags(None, baseDir, content, tags, 'content',
+                                 False)
     # remove replaced emoji
     hashtagsDictCopy = hashtagsDict.copy()
     for tagName, tag in hashtagsDictCopy.items():

session.py

@@ -389,3 +389,67 @@ def postImage(session, attachImageFilename: str, federationList: [],
         if postResult:
             return postResult.text
     return None
+
+
+def downloadImage(session, baseDir: str, url: str,
+                  imageFilename: str, debug: bool,
+                  force: bool = False) -> bool:
+    """Downloads an image
+    """
+    if not url:
+        return None
+
+    # try different image types
+    imageFormats = {
+        'png': 'png',
+        'jpg': 'jpeg',
+        'jpeg': 'jpeg',
+        'gif': 'gif',
+        'svg': 'svg+xml',
+        'webp': 'webp',
+        'avif': 'avif'
+    }
+    sessionHeaders = None
+    for imFormat, mimeType in imageFormats.items():
+        if url.endswith('.' + imFormat) or \
+           '.' + imFormat + '?' in url:
+            sessionHeaders = {
+                'Accept': 'image/' + mimeType
+            }
+            break
+
+    if not sessionHeaders:
+        if debug:
+            print('downloadImage: no session headers')
+        return False
+
+    if not os.path.isfile(imageFilename) or force:
+        try:
+            if debug:
+                print('Downloading image url: ' + url)
+            result = session.get(url,
+                                 headers=sessionHeaders,
+                                 params=None)
+            if result.status_code < 200 or \
+               result.status_code > 202:
+                if debug:
+                    print('Image download failed with status ' +
+                          str(result.status_code))
+                # remove partial download
+                if os.path.isfile(imageFilename):
+                    try:
+                        os.remove(imageFilename)
+                    except BaseException:
+                        print('EX: downloadImage unable to delete ' +
+                              imageFilename)
+                        pass
+            else:
+                with open(imageFilename, 'wb') as f:
+                    f.write(result.content)
+                    if debug:
+                        print('Image downloaded from ' + url)
+                    return True
+        except Exception as e:
+            print('EX: Failed to download image: ' +
+                  str(url) + ' ' + str(e))
+    return False

tests.py

@@ -3379,7 +3379,8 @@ def _testAddEmoji(baseDir: str):
     for tagName, tag in hashtags.items():
         tags.append(tag)
     content = contentModified
-    contentModified = replaceEmojiFromTags(content, tags, 'content')
+    contentModified = \
+        replaceEmojiFromTags(None, baseDir, content, tags, 'content', True)
     # print('contentModified: ' + contentModified)
     assert contentModified == '<p>Emoji 🍋 🍓 🍌</p>'
 

webapp_post.py

@@ -872,7 +872,7 @@ def _getPostTitleAnnounceHtml(baseDir: str,
     # add any emoji to the display name
     if ':' in announceDisplayName:
         announceDisplayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain,
+            addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain,
                                   announceDisplayName, False)
     _logPostTiming(enableTimingLog, postStartTime, '13.3.1')
     titleStr += \
@@ -1054,7 +1054,7 @@ def _getPostTitleReplyHtml(baseDir: str,
         _logPostTiming(enableTimingLog, postStartTime, '13.5')
 
         replyDisplayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain,
+            addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain,
                                   replyDisplayName, False)
         _logPostTiming(enableTimingLog, postStartTime, '13.6')
 
@@ -1316,7 +1316,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
             # add any emoji to the display name
             if ':' in displayName:
                 displayName = \
-                    addEmojiToDisplayName(baseDir, httpPrefix,
+                    addEmojiToDisplayName(session, baseDir, httpPrefix,
                                           nickname, domain,
                                           displayName, False)
 
@@ -1442,7 +1442,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
     if displayName:
         if ':' in displayName:
             displayName = \
-                addEmojiToDisplayName(baseDir, httpPrefix,
+                addEmojiToDisplayName(session, baseDir, httpPrefix,
                                       nickname, domain,
                                       displayName, False)
         titleStr += \
@@ -1740,7 +1740,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
         if postJsonObject['object'].get('summary'):
             cwStr = str(postJsonObject['object']['summary'])
             cwStr = \
-                addEmojiToDisplayName(baseDir, httpPrefix,
+                addEmojiToDisplayName(session, baseDir, httpPrefix,
                                       nickname, domain,
                                       cwStr, False)
             contentStr += \
@@ -1768,9 +1768,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
 
     if postJsonObject['object'].get('tag') and not isPatch:
         contentStr = \
-            replaceEmojiFromTags(contentStr,
+            replaceEmojiFromTags(session, baseDir, contentStr,
                                  postJsonObject['object']['tag'],
-                                 'content')
+                                 'content', False)
 
     if isMuted:
         contentStr = ''

webapp_profile.py

@@ -570,12 +570,12 @@ def htmlProfile(signingPrivateKeyPem: str,
     if not domain:
         return ""
     displayName = \
-        addEmojiToDisplayName(baseDir, httpPrefix,
+        addEmojiToDisplayName(session, baseDir, httpPrefix,
                               nickname, domain,
                               profileJson['name'], True)
     domainFull = getFullDomain(domain, port)
     profileDescription = \
-        addEmojiToDisplayName(baseDir, httpPrefix,
+        addEmojiToDisplayName(session, baseDir, httpPrefix,
                               nickname, domain,
                               profileJson['summary'], False)
     postsButton = 'button'
@@ -2271,7 +2271,7 @@ def _individualFollowAsHtml(signingPrivateKeyPem: str,
         avatarUrl = avatarUrl2
     if displayName:
         displayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix,
+            addEmojiToDisplayName(None, baseDir, httpPrefix,
                                   actorNickname, domain,
                                   displayName, False)
         titleStr = displayName

webapp_search.py

@@ -59,6 +59,7 @@ def htmlSearchEmoji(cssCache: {}, translate: {},
         cssFilename = baseDir + '/epicyon.css'
 
     emojiLookupFilename = baseDir + '/emoji/emoji.json'
+    customEmojiLookupFilename = baseDir + '/emojicustom/emoji.json'
 
     # create header
     instanceTitle = \
@@ -77,6 +78,11 @@ def htmlSearchEmoji(cssCache: {}, translate: {},
 
     emojiJson = loadJson(emojiLookupFilename)
     if emojiJson:
+        if os.path.isfile(customEmojiLookupFilename):
+            customEmojiJson = loadJson(customEmojiLookupFilename)
+            if customEmojiJson:
+                emojiJson = dict(emojiJson, **customEmojiJson)
+
         results = {}
         for emojiName, filename in emojiJson.items():
             if searchStr in emojiName:
@@ -84,20 +90,25 @@ def htmlSearchEmoji(cssCache: {}, translate: {},
         for emojiName, filename in emojiJson.items():
             if emojiName in searchStr:
                 results[emojiName] = filename + '.png'
+
+        if not results:
+            emojiForm += '<center><h5>' + \
+                translate['No results'] + '</h5></center>'
+
         headingShown = False
         emojiForm += '<center>'
         msgStr1 = translate['Copy the text then paste it into your post']
         msgStr2 = ':<img loading="lazy" class="searchEmoji" src="/emoji/'
         for emojiName, filename in results.items():
-            if os.path.isfile(baseDir + '/emoji/' + filename):
+            if not os.path.isfile(baseDir + '/emoji/' + filename):
-                if not headingShown:
+                if not os.path.isfile(baseDir + '/emojicustom/' + filename):
-                    emojiForm += \
+                    continue
-                        '<center><h5>' + msgStr1 + \
+            if not headingShown:
-                        '</h5></center>'
-                    headingShown = True
                 emojiForm += \
-                    '<h3>:' + emojiName + msgStr2 + \
+                    '<center><h5>' + msgStr1 + '</h5></center>'
-                    filename + '"/></h3>'
+                headingShown = True
+            emojiForm += \
+                '<h3>:' + emojiName + msgStr2 + filename + '"/></h3>'
         emojiForm += '</center>'
 
     emojiForm += htmlFooter()

webapp_utils.py

@@ -294,7 +294,7 @@ def updateAvatarImageCache(signingPrivateKeyPem: str,
                         print('avatar image downloaded for ' + actor)
                     return avatarImageFilename.replace(baseDir + '/cache', '')
         except Exception as e:
-            print('WARN: Failed to download avatar image: ' +
+            print('EX: Failed to download avatar image: ' +
                   str(avatarUrl) + ' ' + str(e))
         prof = 'https://www.w3.org/ns/activitystreams'
         if '/channel/' not in actor or '/accounts/' not in actor:
@@ -781,7 +781,7 @@ def loadIndividualPostAsHtmlFromCache(baseDir: str,
         return postHtml
 
 
-def addEmojiToDisplayName(baseDir: str, httpPrefix: str,
+def addEmojiToDisplayName(session, baseDir: str, httpPrefix: str,
                           nickname: str, domain: str,
                           displayName: str, inProfileName: bool) -> str:
     """Adds emoji icons to display names or CW on individual posts
@@ -804,10 +804,14 @@ def addEmojiToDisplayName(baseDir: str, httpPrefix: str,
 #    print('TAG: emoji tags list: ' + str(emojiTagsList))
     if not inProfileName:
         displayName = \
-            replaceEmojiFromTags(displayName, emojiTagsList, 'post header')
+            replaceEmojiFromTags(session, baseDir,
+                                 displayName, emojiTagsList, 'post header',
+                                 False)
     else:
         displayName = \
-            replaceEmojiFromTags(displayName, emojiTagsList, 'profile')
+            replaceEmojiFromTags(session, baseDir,
+                                 displayName, emojiTagsList, 'profile',
+                                 False)
 #    print('TAG: displayName after tags 2: ' + displayName)
 
     # remove any stray emoji