Store custom emoji

blog.py

@@ -164,7 +164,7 @@ def _getBlogReplies(baseDir: str, httpPrefix: str, translate: {},
     return ''
 
 
-def _htmlBlogPostContent(authorized: bool,
+def _htmlBlogPostContent(debug: bool, session, authorized: bool,
                          baseDir: str, httpPrefix: str, translate: {},
                          nickname: str, domain: str, domainFull: str,
                          postJsonObject: {},
@@ -255,9 +255,9 @@ def _htmlBlogPostContent(authorized: bool,
         contentStr = addEmbeddedElements(translate, jsonContent,
                                          peertubeInstances)
         if postJsonObject['object'].get('tag'):
-            contentStr = replaceEmojiFromTags(contentStr,
+            contentStr = replaceEmojiFromTags(session, baseDir, contentStr,
                                               postJsonObject['object']['tag'],
-                                              'content')
+                                              'content', debug)
         if articleAdded:
             blogStr += '<br>' + contentStr + '</article>\n'
         else:
@@ -414,12 +414,13 @@ def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str:
     return content
 
 
-def htmlBlogPost(authorized: bool,
+def htmlBlogPost(session, authorized: bool,
                  baseDir: str, httpPrefix: str, translate: {},
                  nickname: str, domain: str, domainFull: str,
                  postJsonObject: {},
                  peertubeInstances: [],
-                 systemLanguage: str, personCache: {}) -> str:
+                 systemLanguage: str, personCache: {},
+                 debug: bool) -> str:
     """Returns a html blog post
     """
     blogStr = ''
@@ -438,7 +439,7 @@ def htmlBlogPost(authorized: bool,
                                        title, snippet)
     _htmlBlogRemoveCwButton(blogStr, translate)
 
-    blogStr += _htmlBlogPostContent(authorized, baseDir,
+    blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir,
                                     httpPrefix, translate,
                                     nickname, domain,
                                     domainFull, postJsonObject,
@@ -473,7 +474,7 @@ def htmlBlogPage(authorized: bool, session,
                  nickname: str, domain: str, port: int,
                  noOfItems: int, pageNumber: int,
                  peertubeInstances: [], systemLanguage: str,
-                 personCache: {}) -> str:
+                 personCache: {}, debug: bool) -> str:
     """Returns a html blog page containing posts
     """
     if ' ' in nickname or '@' in nickname or \
@@ -530,7 +531,7 @@ def htmlBlogPage(authorized: bool, session,
         if item['type'] != 'Create':
             continue
 
-        blogStr += _htmlBlogPostContent(authorized, baseDir,
+        blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir,
                                         httpPrefix, translate,
                                         nickname, domain,
                                         domainFull, item,
@@ -696,7 +697,7 @@ def htmlBlogView(authorized: bool,
                  translate: {}, domain: str, port: int,
                  noOfItems: int,
                  peertubeInstances: [], systemLanguage: str,
-                 personCache: {}) -> str:
+                 personCache: {}, debug: bool) -> str:
     """Show the blog main page
     """
     blogStr = ''
@@ -715,7 +716,7 @@ def htmlBlogView(authorized: bool,
                                 baseDir, httpPrefix, translate,
                                 nickname, domain, port,
                                 noOfItems, 1, peertubeInstances,
-                                systemLanguage, personCache)
+                                systemLanguage, personCache, debug)
 
     domainFull = getFullDomain(domain, port)
 

content.py

@@ -16,6 +16,7 @@ from utils import removeDomainPort
 from utils import isValidLanguage
 from utils import getImageExtensions
 from utils import loadJson
+from utils import saveJson
 from utils import fileLastModified
 from utils import getLinkPrefixes
 from utils import dangerousMarkup
@@ -26,6 +27,7 @@ from utils import isfloat
 from utils import getCurrencies
 from utils import removeHtml
 from petnames import getPetName
+from session import downloadImage
 
 
 def removeHtmlTag(htmlStr: str, tag: str) -> str:
@@ -239,7 +241,40 @@ def switchWords(baseDir: str, nickname: str, domain: str, content: str,
     return content
 
 
-def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
+def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str,
+                     debug: bool) -> None:
+    """Saves custom emoji to file
+    """
+    if not session:
+        return
+    if '.' not in url:
+        return
+    ext = url.split('.')[-1]
+    if ext != 'png':
+        print('Custom emoji is wrong format ' + url)
+        return
+    emojiName = emojiName.replace(':').strip()
+    customEmojiDir = baseDir + '/emojicustom'
+    if not os.path.isdir(customEmojiDir):
+        os.mkdir(customEmojiDir)
+    emojiImageFilename = customEmojiDir + '/' + emojiName + '.' + ext
+    if not downloadImage(session, baseDir, url,
+                         emojiImageFilename, debug, False):
+        return
+    emojiJsonFilename = customEmojiDir + '/emoji.json'
+    emojiJson = {}
+    if os.path.isfile(emojiJsonFilename):
+        emojiJson = loadJson(emojiJsonFilename, 0, 1)
+        if not emojiJson:
+            emojiJson = {}
+    if not emojiJson.get(emojiName):
+        emojiJson[emojiName] = emojiName
+        saveJson(emojiJson, emojiJsonFilename)
+
+
+def replaceEmojiFromTags(session, baseDir: str,
+                         content: str, tag: [], messageType: str,
+                         debug: bool) -> str:
     """Uses the tags to replace :emoji: with html image markup
     """
     for tagItem in tag:
@@ -265,12 +300,14 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
                         iconName = iconName.split('.')[0]
                         # see https://unicode.org/
                         # emoji/charts/full-emoji-list.html
+                        replaced = False
                         if '-' not in iconName:
                             # a single code
                             try:
                                 replaceChar = chr(int("0x" + iconName, 16))
                                 content = content.replace(tagItem['name'],
                                                           replaceChar)
+                                replaced = True
                             except BaseException:
                                 print('EX: replaceEmojiFromTags ' +
                                       'no conversion of ' +
@@ -278,6 +315,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
                                       tagItem['name'] + ' ' +
                                       tagItem['icon']['url'])
                                 pass
+                            if not replaced:
+                                _saveCustomEmoji(session, baseDir,
+                                                 tagItem['name'],
+                                                 tagItem['icon']['url'],
+                                                 debug)
                         else:
                             # sequence of codes
                             iconCodes = iconName.split('-')
@@ -286,6 +328,7 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
                                 try:
                                     iconCodeSequence += chr(int("0x" +
                                                                 icode, 16))
+                                    replaced = True
                                 except BaseException:
                                     iconCodeSequence = ''
                                     print('EX: replaceEmojiFromTags ' +
@@ -294,6 +337,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
                                           tagItem['name'] + ' ' +
                                           tagItem['icon']['url'])
                                     break
+                                if not replaced:
+                                    _saveCustomEmoji(session, baseDir,
+                                                     tagItem['name'],
+                                                     tagItem['icon']['url'],
+                                                     debug)
                             if iconCodeSequence:
                                 content = content.replace(tagItem['name'],
                                                           iconCodeSequence)

daemon.py

@@ -11379,7 +11379,8 @@ class PubServer(BaseHTTPRequestHandler):
                            maxPostsInBlogsFeed, pageNumber,
                            self.server.peertubeInstances,
                            self.server.systemLanguage,
-                           self.server.personCache)
+                           self.server.personCache,
+                           self.server.debug)
         if msg is not None:
             msg = msg.encode('utf-8')
             msglen = len(msg)
@@ -12983,7 +12984,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    maxPostsInBlogsFeed,
                                    self.server.peertubeInstances,
                                    self.server.systemLanguage,
-                                   self.server.personCache)
+                                   self.server.personCache,
+                                   self.server.debug)
                 if msg is not None:
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
@@ -13076,7 +13078,8 @@ class PubServer(BaseHTTPRequestHandler):
             if blogFilename and nickname:
                 postJsonObject = loadJson(blogFilename)
                 if isBlogPost(postJsonObject):
-                    msg = htmlBlogPost(authorized,
+                    msg = htmlBlogPost(self.server.session,
+                                       authorized,
                                        self.server.baseDir,
                                        self.server.httpPrefix,
                                        self.server.translate,
@@ -13085,7 +13088,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        postJsonObject,
                                        self.server.peertubeInstances,
                                        self.server.systemLanguage,
-                                       self.server.personCache)
+                                       self.server.personCache,
+                                       self.server.debug)
                     if msg is not None:
                         msg = msg.encode('utf-8')
                         msglen = len(msg)
@@ -15538,8 +15542,11 @@ class PubServer(BaseHTTPRequestHandler):
                             tags.append(tag)
                         # get list of tags
                         fields['message'] = \
-                            replaceEmojiFromTags(fields['message'],
+                            replaceEmojiFromTags(self.server.session,
-                                                 tags, 'content')
+                                                 self.server.baseDir,
+                                                 fields['message'],
+                                                 tags, 'content',
+                                                 self.server.debug)
 
                         postJsonObject['object']['content'] = fields['message']
                         contentMap = postJsonObject['object']['contentMap']

posts.py

@@ -1292,7 +1292,8 @@ def _createPostModReport(baseDir: str,
         modFile.write(newPostId + '\n')
 
 
-def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
+def _createPostBase(baseDir: str,
+                    nickname: str, domain: str, port: int,
                     toUrl: str, ccUrl: str, httpPrefix: str, content: str,
                     followersOnly: bool, saveToFile: bool,
                     clientToServer: bool, commentsEnabled: bool,
@@ -1345,7 +1346,9 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int,
         tags.append(tag)
     # get list of tags
     if nickname != 'news':
-        content = replaceEmojiFromTags(content, tags, 'content')
+        content = \
+            replaceEmojiFromTags(None, baseDir, content, tags, 'content',
+                                 False)
     # remove replaced emoji
     hashtagsDictCopy = hashtagsDict.copy()
     for tagName, tag in hashtagsDictCopy.items():

session.py

@@ -389,3 +389,64 @@ def postImage(session, attachImageFilename: str, federationList: [],
         if postResult:
             return postResult.text
     return None
+
+
+def downloadImage(session, baseDir: str, url: str,
+                  imageFilename: str, debug: bool,
+                  force: bool = False) -> bool:
+    """Downloads an image
+    """
+    if not url:
+        return None
+
+    # try different image types
+    imageFormats = {
+        'png': 'png',
+        'jpg': 'jpeg',
+        'jpeg': 'jpeg',
+        'gif': 'gif',
+        'svg': 'svg+xml',
+        'webp': 'webp',
+        'avif': 'avif'
+    }
+    sessionHeaders = None
+    for imFormat, mimeType in imageFormats.items():
+        if url.endswith('.' + imFormat) or \
+           '.' + imFormat + '?' in url:
+            sessionHeaders = {
+                'Accept': 'image/' + mimeType
+            }
+
+    if not sessionHeaders:
+        return False
+
+    if not os.path.isfile(imageFilename) or force:
+        try:
+            if debug:
+                print('Downloading image url: ' + url)
+            result = session.get(url,
+                                 headers=sessionHeaders,
+                                 params=None)
+            if result.status_code < 200 or \
+               result.status_code > 202:
+                if debug:
+                    print('Image download failed with status ' +
+                          str(result.status_code))
+                # remove partial download
+                if os.path.isfile(imageFilename):
+                    try:
+                        os.remove(imageFilename)
+                    except BaseException:
+                        print('EX: downloadImage unable to delete ' +
+                              imageFilename)
+                        pass
+            else:
+                with open(imageFilename, 'wb') as f:
+                    f.write(result.content)
+                    if debug:
+                        print('Image downloaded from ' + url)
+                    return True
+        except Exception as e:
+            print('EX: Failed to download image: ' +
+                  str(url) + ' ' + str(e))
+    return False

tests.py

@@ -3379,7 +3379,8 @@ def _testAddEmoji(baseDir: str):
     for tagName, tag in hashtags.items():
         tags.append(tag)
     content = contentModified
-    contentModified = replaceEmojiFromTags(content, tags, 'content')
+    contentModified = \
+        replaceEmojiFromTags(None, baseDir, content, tags, 'content', True)
     # print('contentModified: ' + contentModified)
     assert contentModified == '<p>Emoji 🍋 🍓 🍌</p>'
 

webapp_post.py

@@ -872,7 +872,7 @@ def _getPostTitleAnnounceHtml(baseDir: str,
     # add any emoji to the display name
     if ':' in announceDisplayName:
         announceDisplayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain,
+            addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain,
                                   announceDisplayName, False)
     _logPostTiming(enableTimingLog, postStartTime, '13.3.1')
     titleStr += \
@@ -1054,7 +1054,7 @@ def _getPostTitleReplyHtml(baseDir: str,
         _logPostTiming(enableTimingLog, postStartTime, '13.5')
 
         replyDisplayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain,
+            addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain,
                                   replyDisplayName, False)
         _logPostTiming(enableTimingLog, postStartTime, '13.6')
 
@@ -1316,7 +1316,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
             # add any emoji to the display name
             if ':' in displayName:
                 displayName = \
-                    addEmojiToDisplayName(baseDir, httpPrefix,
+                    addEmojiToDisplayName(session, baseDir, httpPrefix,
                                           nickname, domain,
                                           displayName, False)
 
@@ -1442,7 +1442,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
     if displayName:
         if ':' in displayName:
             displayName = \
-                addEmojiToDisplayName(baseDir, httpPrefix,
+                addEmojiToDisplayName(session, baseDir, httpPrefix,
                                       nickname, domain,
                                       displayName, False)
         titleStr += \
@@ -1740,7 +1740,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
         if postJsonObject['object'].get('summary'):
             cwStr = str(postJsonObject['object']['summary'])
             cwStr = \
-                addEmojiToDisplayName(baseDir, httpPrefix,
+                addEmojiToDisplayName(session, baseDir, httpPrefix,
                                       nickname, domain,
                                       cwStr, False)
             contentStr += \
@@ -1768,9 +1768,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
 
     if postJsonObject['object'].get('tag') and not isPatch:
         contentStr = \
-            replaceEmojiFromTags(contentStr,
+            replaceEmojiFromTags(session, baseDir, contentStr,
                                  postJsonObject['object']['tag'],
-                                 'content')
+                                 'content', False)
 
     if isMuted:
         contentStr = ''

webapp_profile.py

@@ -570,12 +570,12 @@ def htmlProfile(signingPrivateKeyPem: str,
     if not domain:
         return ""
     displayName = \
-        addEmojiToDisplayName(baseDir, httpPrefix,
+        addEmojiToDisplayName(session, baseDir, httpPrefix,
                               nickname, domain,
                               profileJson['name'], True)
     domainFull = getFullDomain(domain, port)
     profileDescription = \
-        addEmojiToDisplayName(baseDir, httpPrefix,
+        addEmojiToDisplayName(session, baseDir, httpPrefix,
                               nickname, domain,
                               profileJson['summary'], False)
     postsButton = 'button'
@@ -2271,7 +2271,7 @@ def _individualFollowAsHtml(signingPrivateKeyPem: str,
         avatarUrl = avatarUrl2
     if displayName:
         displayName = \
-            addEmojiToDisplayName(baseDir, httpPrefix,
+            addEmojiToDisplayName(None, baseDir, httpPrefix,
                                   actorNickname, domain,
                                   displayName, False)
         titleStr = displayName

webapp_utils.py

@@ -294,7 +294,7 @@ def updateAvatarImageCache(signingPrivateKeyPem: str,
                         print('avatar image downloaded for ' + actor)
                     return avatarImageFilename.replace(baseDir + '/cache', '')
         except Exception as e:
-            print('WARN: Failed to download avatar image: ' +
+            print('EX: Failed to download avatar image: ' +
                   str(avatarUrl) + ' ' + str(e))
         prof = 'https://www.w3.org/ns/activitystreams'
         if '/channel/' not in actor or '/accounts/' not in actor:
@@ -781,7 +781,7 @@ def loadIndividualPostAsHtmlFromCache(baseDir: str,
         return postHtml
 
 
-def addEmojiToDisplayName(baseDir: str, httpPrefix: str,
+def addEmojiToDisplayName(session, baseDir: str, httpPrefix: str,
                           nickname: str, domain: str,
                           displayName: str, inProfileName: bool) -> str:
     """Adds emoji icons to display names or CW on individual posts
@@ -804,10 +804,14 @@ def addEmojiToDisplayName(baseDir: str, httpPrefix: str,
 #    print('TAG: emoji tags list: ' + str(emojiTagsList))
     if not inProfileName:
         displayName = \
-            replaceEmojiFromTags(displayName, emojiTagsList, 'post header')
+            replaceEmojiFromTags(session, baseDir,
+                                 displayName, emojiTagsList, 'post header',
+                                 False)
     else:
         displayName = \
-            replaceEmojiFromTags(displayName, emojiTagsList, 'profile')
+            replaceEmojiFromTags(session, baseDir,
+                                 displayName, emojiTagsList, 'profile',
+                                 False)
 #    print('TAG: displayName after tags 2: ' + displayName)
 
     # remove any stray emoji