From 4e81096b32389c6d89bde5df61ef4a1fe97a10d0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 17:12:17 +0000 Subject: [PATCH] Store custom emoji --- blog.py | 21 ++++++++-------- content.py | 50 +++++++++++++++++++++++++++++++++++++- daemon.py | 19 ++++++++++----- posts.py | 7 ++++-- session.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ tests.py | 3 ++- webapp_post.py | 14 +++++------ webapp_profile.py | 6 ++--- webapp_utils.py | 12 ++++++---- 9 files changed, 159 insertions(+), 34 deletions(-) diff --git a/blog.py b/blog.py index fbdcadad0..d273f1c4c 100644 --- a/blog.py +++ b/blog.py @@ -164,7 +164,7 @@ def _getBlogReplies(baseDir: str, httpPrefix: str, translate: {}, return '' -def _htmlBlogPostContent(authorized: bool, +def _htmlBlogPostContent(debug: bool, session, authorized: bool, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, domainFull: str, postJsonObject: {}, @@ -255,9 +255,9 @@ def _htmlBlogPostContent(authorized: bool, contentStr = addEmbeddedElements(translate, jsonContent, peertubeInstances) if postJsonObject['object'].get('tag'): - contentStr = replaceEmojiFromTags(contentStr, + contentStr = replaceEmojiFromTags(session, baseDir, contentStr, postJsonObject['object']['tag'], - 'content') + 'content', debug) if articleAdded: blogStr += '
' + contentStr + '\n' else: @@ -414,12 +414,13 @@ def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str: return content -def htmlBlogPost(authorized: bool, +def htmlBlogPost(session, authorized: bool, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, domainFull: str, postJsonObject: {}, peertubeInstances: [], - systemLanguage: str, personCache: {}) -> str: + systemLanguage: str, personCache: {}, + debug: bool) -> str: """Returns a html blog post """ blogStr = '' @@ -438,7 +439,7 @@ def htmlBlogPost(authorized: bool, title, snippet) _htmlBlogRemoveCwButton(blogStr, translate) - blogStr += _htmlBlogPostContent(authorized, baseDir, + blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir, httpPrefix, translate, nickname, domain, domainFull, postJsonObject, @@ -473,7 +474,7 @@ def htmlBlogPage(authorized: bool, session, nickname: str, domain: str, port: int, noOfItems: int, pageNumber: int, peertubeInstances: [], systemLanguage: str, - personCache: {}) -> str: + personCache: {}, debug: bool) -> str: """Returns a html blog page containing posts """ if ' ' in nickname or '@' in nickname or \ @@ -530,7 +531,7 @@ def htmlBlogPage(authorized: bool, session, if item['type'] != 'Create': continue - blogStr += _htmlBlogPostContent(authorized, baseDir, + blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir, httpPrefix, translate, nickname, domain, domainFull, item, @@ -696,7 +697,7 @@ def htmlBlogView(authorized: bool, translate: {}, domain: str, port: int, noOfItems: int, peertubeInstances: [], systemLanguage: str, - personCache: {}) -> str: + personCache: {}, debug: bool) -> str: """Show the blog main page """ blogStr = '' @@ -715,7 +716,7 @@ def htmlBlogView(authorized: bool, baseDir, httpPrefix, translate, nickname, domain, port, noOfItems, 1, peertubeInstances, - systemLanguage, personCache) + systemLanguage, personCache, debug) domainFull = getFullDomain(domain, port) diff --git a/content.py b/content.py index c15b10848..d223e4706 100644 --- a/content.py +++ b/content.py @@ -16,6 +16,7 @@ from utils import removeDomainPort from utils import isValidLanguage from utils import getImageExtensions from utils import loadJson +from utils import saveJson from utils import fileLastModified from utils import getLinkPrefixes from utils import dangerousMarkup @@ -26,6 +27,7 @@ from utils import isfloat from utils import getCurrencies from utils import removeHtml from petnames import getPetName +from session import downloadImage def removeHtmlTag(htmlStr: str, tag: str) -> str: @@ -239,7 +241,40 @@ def switchWords(baseDir: str, nickname: str, domain: str, content: str, return content -def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: +def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, + debug: bool) -> None: + """Saves custom emoji to file + """ + if not session: + return + if '.' not in url: + return + ext = url.split('.')[-1] + if ext != 'png': + print('Custom emoji is wrong format ' + url) + return + emojiName = emojiName.replace(':').strip() + customEmojiDir = baseDir + '/emojicustom' + if not os.path.isdir(customEmojiDir): + os.mkdir(customEmojiDir) + emojiImageFilename = customEmojiDir + '/' + emojiName + '.' + ext + if not downloadImage(session, baseDir, url, + emojiImageFilename, debug, False): + return + emojiJsonFilename = customEmojiDir + '/emoji.json' + emojiJson = {} + if os.path.isfile(emojiJsonFilename): + emojiJson = loadJson(emojiJsonFilename, 0, 1) + if not emojiJson: + emojiJson = {} + if not emojiJson.get(emojiName): + emojiJson[emojiName] = emojiName + saveJson(emojiJson, emojiJsonFilename) + + +def replaceEmojiFromTags(session, baseDir: str, + content: str, tag: [], messageType: str, + debug: bool) -> str: """Uses the tags to replace :emoji: with html image markup """ for tagItem in tag: @@ -265,12 +300,14 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: iconName = iconName.split('.')[0] # see https://unicode.org/ # emoji/charts/full-emoji-list.html + replaced = False if '-' not in iconName: # a single code try: replaceChar = chr(int("0x" + iconName, 16)) content = content.replace(tagItem['name'], replaceChar) + replaced = True except BaseException: print('EX: replaceEmojiFromTags ' + 'no conversion of ' + @@ -278,6 +315,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: tagItem['name'] + ' ' + tagItem['icon']['url']) pass + if not replaced: + _saveCustomEmoji(session, baseDir, + tagItem['name'], + tagItem['icon']['url'], + debug) else: # sequence of codes iconCodes = iconName.split('-') @@ -286,6 +328,7 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: try: iconCodeSequence += chr(int("0x" + icode, 16)) + replaced = True except BaseException: iconCodeSequence = '' print('EX: replaceEmojiFromTags ' + @@ -294,6 +337,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: tagItem['name'] + ' ' + tagItem['icon']['url']) break + if not replaced: + _saveCustomEmoji(session, baseDir, + tagItem['name'], + tagItem['icon']['url'], + debug) if iconCodeSequence: content = content.replace(tagItem['name'], iconCodeSequence) diff --git a/daemon.py b/daemon.py index 347e895b8..74e0fc5e2 100644 --- a/daemon.py +++ b/daemon.py @@ -11379,7 +11379,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, pageNumber, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -12983,7 +12984,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -13076,7 +13078,8 @@ class PubServer(BaseHTTPRequestHandler): if blogFilename and nickname: postJsonObject = loadJson(blogFilename) if isBlogPost(postJsonObject): - msg = htmlBlogPost(authorized, + msg = htmlBlogPost(self.server.session, + authorized, self.server.baseDir, self.server.httpPrefix, self.server.translate, @@ -13085,7 +13088,8 @@ class PubServer(BaseHTTPRequestHandler): postJsonObject, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -15538,8 +15542,11 @@ class PubServer(BaseHTTPRequestHandler): tags.append(tag) # get list of tags fields['message'] = \ - replaceEmojiFromTags(fields['message'], - tags, 'content') + replaceEmojiFromTags(self.server.session, + self.server.baseDir, + fields['message'], + tags, 'content', + self.server.debug) postJsonObject['object']['content'] = fields['message'] contentMap = postJsonObject['object']['contentMap'] diff --git a/posts.py b/posts.py index 71a87bbc9..9df4a2505 100644 --- a/posts.py +++ b/posts.py @@ -1292,7 +1292,8 @@ def _createPostModReport(baseDir: str, modFile.write(newPostId + '\n') -def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, +def _createPostBase(baseDir: str, + nickname: str, domain: str, port: int, toUrl: str, ccUrl: str, httpPrefix: str, content: str, followersOnly: bool, saveToFile: bool, clientToServer: bool, commentsEnabled: bool, @@ -1345,7 +1346,9 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, tags.append(tag) # get list of tags if nickname != 'news': - content = replaceEmojiFromTags(content, tags, 'content') + content = \ + replaceEmojiFromTags(None, baseDir, content, tags, 'content', + False) # remove replaced emoji hashtagsDictCopy = hashtagsDict.copy() for tagName, tag in hashtagsDictCopy.items(): diff --git a/session.py b/session.py index 128da7341..e1ef583cd 100644 --- a/session.py +++ b/session.py @@ -389,3 +389,64 @@ def postImage(session, attachImageFilename: str, federationList: [], if postResult: return postResult.text return None + + +def downloadImage(session, baseDir: str, url: str, + imageFilename: str, debug: bool, + force: bool = False) -> bool: + """Downloads an image + """ + if not url: + return None + + # try different image types + imageFormats = { + 'png': 'png', + 'jpg': 'jpeg', + 'jpeg': 'jpeg', + 'gif': 'gif', + 'svg': 'svg+xml', + 'webp': 'webp', + 'avif': 'avif' + } + sessionHeaders = None + for imFormat, mimeType in imageFormats.items(): + if url.endswith('.' + imFormat) or \ + '.' + imFormat + '?' in url: + sessionHeaders = { + 'Accept': 'image/' + mimeType + } + + if not sessionHeaders: + return False + + if not os.path.isfile(imageFilename) or force: + try: + if debug: + print('Downloading image url: ' + url) + result = session.get(url, + headers=sessionHeaders, + params=None) + if result.status_code < 200 or \ + result.status_code > 202: + if debug: + print('Image download failed with status ' + + str(result.status_code)) + # remove partial download + if os.path.isfile(imageFilename): + try: + os.remove(imageFilename) + except BaseException: + print('EX: downloadImage unable to delete ' + + imageFilename) + pass + else: + with open(imageFilename, 'wb') as f: + f.write(result.content) + if debug: + print('Image downloaded from ' + url) + return True + except Exception as e: + print('EX: Failed to download image: ' + + str(url) + ' ' + str(e)) + return False diff --git a/tests.py b/tests.py index 28870da73..876594409 100644 --- a/tests.py +++ b/tests.py @@ -3379,7 +3379,8 @@ def _testAddEmoji(baseDir: str): for tagName, tag in hashtags.items(): tags.append(tag) content = contentModified - contentModified = replaceEmojiFromTags(content, tags, 'content') + contentModified = \ + replaceEmojiFromTags(None, baseDir, content, tags, 'content', True) # print('contentModified: ' + contentModified) assert contentModified == '

Emoji 🍋 🍓 🍌

' diff --git a/webapp_post.py b/webapp_post.py index e05d521a7..60a67fe18 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -872,7 +872,7 @@ def _getPostTitleAnnounceHtml(baseDir: str, # add any emoji to the display name if ':' in announceDisplayName: announceDisplayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain, + addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain, announceDisplayName, False) _logPostTiming(enableTimingLog, postStartTime, '13.3.1') titleStr += \ @@ -1054,7 +1054,7 @@ def _getPostTitleReplyHtml(baseDir: str, _logPostTiming(enableTimingLog, postStartTime, '13.5') replyDisplayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain, + addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain, replyDisplayName, False) _logPostTiming(enableTimingLog, postStartTime, '13.6') @@ -1316,7 +1316,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, # add any emoji to the display name if ':' in displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, displayName, False) @@ -1442,7 +1442,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if displayName: if ':' in displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, displayName, False) titleStr += \ @@ -1740,7 +1740,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if postJsonObject['object'].get('summary'): cwStr = str(postJsonObject['object']['summary']) cwStr = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, cwStr, False) contentStr += \ @@ -1768,9 +1768,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if postJsonObject['object'].get('tag') and not isPatch: contentStr = \ - replaceEmojiFromTags(contentStr, + replaceEmojiFromTags(session, baseDir, contentStr, postJsonObject['object']['tag'], - 'content') + 'content', False) if isMuted: contentStr = '' diff --git a/webapp_profile.py b/webapp_profile.py index 34d1646c2..3113b495b 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -570,12 +570,12 @@ def htmlProfile(signingPrivateKeyPem: str, if not domain: return "" displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, profileJson['name'], True) domainFull = getFullDomain(domain, port) profileDescription = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, profileJson['summary'], False) postsButton = 'button' @@ -2271,7 +2271,7 @@ def _individualFollowAsHtml(signingPrivateKeyPem: str, avatarUrl = avatarUrl2 if displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(None, baseDir, httpPrefix, actorNickname, domain, displayName, False) titleStr = displayName diff --git a/webapp_utils.py b/webapp_utils.py index a03c4895b..1865ff6d9 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -294,7 +294,7 @@ def updateAvatarImageCache(signingPrivateKeyPem: str, print('avatar image downloaded for ' + actor) return avatarImageFilename.replace(baseDir + '/cache', '') except Exception as e: - print('WARN: Failed to download avatar image: ' + + print('EX: Failed to download avatar image: ' + str(avatarUrl) + ' ' + str(e)) prof = 'https://www.w3.org/ns/activitystreams' if '/channel/' not in actor or '/accounts/' not in actor: @@ -781,7 +781,7 @@ def loadIndividualPostAsHtmlFromCache(baseDir: str, return postHtml -def addEmojiToDisplayName(baseDir: str, httpPrefix: str, +def addEmojiToDisplayName(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, displayName: str, inProfileName: bool) -> str: """Adds emoji icons to display names or CW on individual posts @@ -804,10 +804,14 @@ def addEmojiToDisplayName(baseDir: str, httpPrefix: str, # print('TAG: emoji tags list: ' + str(emojiTagsList)) if not inProfileName: displayName = \ - replaceEmojiFromTags(displayName, emojiTagsList, 'post header') + replaceEmojiFromTags(session, baseDir, + displayName, emojiTagsList, 'post header', + False) else: displayName = \ - replaceEmojiFromTags(displayName, emojiTagsList, 'profile') + replaceEmojiFromTags(session, baseDir, + displayName, emojiTagsList, 'profile', + False) # print('TAG: displayName after tags 2: ' + displayName) # remove any stray emoji