From 4e81096b32389c6d89bde5df61ef4a1fe97a10d0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 17:12:17 +0000 Subject: [PATCH 01/20] Store custom emoji --- blog.py | 21 ++++++++-------- content.py | 50 +++++++++++++++++++++++++++++++++++++- daemon.py | 19 ++++++++++----- posts.py | 7 ++++-- session.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ tests.py | 3 ++- webapp_post.py | 14 +++++------ webapp_profile.py | 6 ++--- webapp_utils.py | 12 ++++++---- 9 files changed, 159 insertions(+), 34 deletions(-) diff --git a/blog.py b/blog.py index fbdcadad0..d273f1c4c 100644 --- a/blog.py +++ b/blog.py @@ -164,7 +164,7 @@ def _getBlogReplies(baseDir: str, httpPrefix: str, translate: {}, return '' -def _htmlBlogPostContent(authorized: bool, +def _htmlBlogPostContent(debug: bool, session, authorized: bool, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, domainFull: str, postJsonObject: {}, @@ -255,9 +255,9 @@ def _htmlBlogPostContent(authorized: bool, contentStr = addEmbeddedElements(translate, jsonContent, peertubeInstances) if postJsonObject['object'].get('tag'): - contentStr = replaceEmojiFromTags(contentStr, + contentStr = replaceEmojiFromTags(session, baseDir, contentStr, postJsonObject['object']['tag'], - 'content') + 'content', debug) if articleAdded: blogStr += '
' + contentStr + '\n' else: @@ -414,12 +414,13 @@ def _getSnippetFromBlogContent(postJsonObject: {}, systemLanguage: str) -> str: return content -def htmlBlogPost(authorized: bool, +def htmlBlogPost(session, authorized: bool, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, domainFull: str, postJsonObject: {}, peertubeInstances: [], - systemLanguage: str, personCache: {}) -> str: + systemLanguage: str, personCache: {}, + debug: bool) -> str: """Returns a html blog post """ blogStr = '' @@ -438,7 +439,7 @@ def htmlBlogPost(authorized: bool, title, snippet) _htmlBlogRemoveCwButton(blogStr, translate) - blogStr += _htmlBlogPostContent(authorized, baseDir, + blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir, httpPrefix, translate, nickname, domain, domainFull, postJsonObject, @@ -473,7 +474,7 @@ def htmlBlogPage(authorized: bool, session, nickname: str, domain: str, port: int, noOfItems: int, pageNumber: int, peertubeInstances: [], systemLanguage: str, - personCache: {}) -> str: + personCache: {}, debug: bool) -> str: """Returns a html blog page containing posts """ if ' ' in nickname or '@' in nickname or \ @@ -530,7 +531,7 @@ def htmlBlogPage(authorized: bool, session, if item['type'] != 'Create': continue - blogStr += _htmlBlogPostContent(authorized, baseDir, + blogStr += _htmlBlogPostContent(debug, session, authorized, baseDir, httpPrefix, translate, nickname, domain, domainFull, item, @@ -696,7 +697,7 @@ def htmlBlogView(authorized: bool, translate: {}, domain: str, port: int, noOfItems: int, peertubeInstances: [], systemLanguage: str, - personCache: {}) -> str: + personCache: {}, debug: bool) -> str: """Show the blog main page """ blogStr = '' @@ -715,7 +716,7 @@ def htmlBlogView(authorized: bool, baseDir, httpPrefix, translate, nickname, domain, port, noOfItems, 1, peertubeInstances, - systemLanguage, personCache) + systemLanguage, personCache, debug) domainFull = getFullDomain(domain, port) diff --git a/content.py b/content.py index c15b10848..d223e4706 100644 --- a/content.py +++ b/content.py @@ -16,6 +16,7 @@ from utils import removeDomainPort from utils import isValidLanguage from utils import getImageExtensions from utils import loadJson +from utils import saveJson from utils import fileLastModified from utils import getLinkPrefixes from utils import dangerousMarkup @@ -26,6 +27,7 @@ from utils import isfloat from utils import getCurrencies from utils import removeHtml from petnames import getPetName +from session import downloadImage def removeHtmlTag(htmlStr: str, tag: str) -> str: @@ -239,7 +241,40 @@ def switchWords(baseDir: str, nickname: str, domain: str, content: str, return content -def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: +def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, + debug: bool) -> None: + """Saves custom emoji to file + """ + if not session: + return + if '.' not in url: + return + ext = url.split('.')[-1] + if ext != 'png': + print('Custom emoji is wrong format ' + url) + return + emojiName = emojiName.replace(':').strip() + customEmojiDir = baseDir + '/emojicustom' + if not os.path.isdir(customEmojiDir): + os.mkdir(customEmojiDir) + emojiImageFilename = customEmojiDir + '/' + emojiName + '.' + ext + if not downloadImage(session, baseDir, url, + emojiImageFilename, debug, False): + return + emojiJsonFilename = customEmojiDir + '/emoji.json' + emojiJson = {} + if os.path.isfile(emojiJsonFilename): + emojiJson = loadJson(emojiJsonFilename, 0, 1) + if not emojiJson: + emojiJson = {} + if not emojiJson.get(emojiName): + emojiJson[emojiName] = emojiName + saveJson(emojiJson, emojiJsonFilename) + + +def replaceEmojiFromTags(session, baseDir: str, + content: str, tag: [], messageType: str, + debug: bool) -> str: """Uses the tags to replace :emoji: with html image markup """ for tagItem in tag: @@ -265,12 +300,14 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: iconName = iconName.split('.')[0] # see https://unicode.org/ # emoji/charts/full-emoji-list.html + replaced = False if '-' not in iconName: # a single code try: replaceChar = chr(int("0x" + iconName, 16)) content = content.replace(tagItem['name'], replaceChar) + replaced = True except BaseException: print('EX: replaceEmojiFromTags ' + 'no conversion of ' + @@ -278,6 +315,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: tagItem['name'] + ' ' + tagItem['icon']['url']) pass + if not replaced: + _saveCustomEmoji(session, baseDir, + tagItem['name'], + tagItem['icon']['url'], + debug) else: # sequence of codes iconCodes = iconName.split('-') @@ -286,6 +328,7 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: try: iconCodeSequence += chr(int("0x" + icode, 16)) + replaced = True except BaseException: iconCodeSequence = '' print('EX: replaceEmojiFromTags ' + @@ -294,6 +337,11 @@ def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: tagItem['name'] + ' ' + tagItem['icon']['url']) break + if not replaced: + _saveCustomEmoji(session, baseDir, + tagItem['name'], + tagItem['icon']['url'], + debug) if iconCodeSequence: content = content.replace(tagItem['name'], iconCodeSequence) diff --git a/daemon.py b/daemon.py index 347e895b8..74e0fc5e2 100644 --- a/daemon.py +++ b/daemon.py @@ -11379,7 +11379,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, pageNumber, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -12983,7 +12984,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -13076,7 +13078,8 @@ class PubServer(BaseHTTPRequestHandler): if blogFilename and nickname: postJsonObject = loadJson(blogFilename) if isBlogPost(postJsonObject): - msg = htmlBlogPost(authorized, + msg = htmlBlogPost(self.server.session, + authorized, self.server.baseDir, self.server.httpPrefix, self.server.translate, @@ -13085,7 +13088,8 @@ class PubServer(BaseHTTPRequestHandler): postJsonObject, self.server.peertubeInstances, self.server.systemLanguage, - self.server.personCache) + self.server.personCache, + self.server.debug) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -15538,8 +15542,11 @@ class PubServer(BaseHTTPRequestHandler): tags.append(tag) # get list of tags fields['message'] = \ - replaceEmojiFromTags(fields['message'], - tags, 'content') + replaceEmojiFromTags(self.server.session, + self.server.baseDir, + fields['message'], + tags, 'content', + self.server.debug) postJsonObject['object']['content'] = fields['message'] contentMap = postJsonObject['object']['contentMap'] diff --git a/posts.py b/posts.py index 71a87bbc9..9df4a2505 100644 --- a/posts.py +++ b/posts.py @@ -1292,7 +1292,8 @@ def _createPostModReport(baseDir: str, modFile.write(newPostId + '\n') -def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, +def _createPostBase(baseDir: str, + nickname: str, domain: str, port: int, toUrl: str, ccUrl: str, httpPrefix: str, content: str, followersOnly: bool, saveToFile: bool, clientToServer: bool, commentsEnabled: bool, @@ -1345,7 +1346,9 @@ def _createPostBase(baseDir: str, nickname: str, domain: str, port: int, tags.append(tag) # get list of tags if nickname != 'news': - content = replaceEmojiFromTags(content, tags, 'content') + content = \ + replaceEmojiFromTags(None, baseDir, content, tags, 'content', + False) # remove replaced emoji hashtagsDictCopy = hashtagsDict.copy() for tagName, tag in hashtagsDictCopy.items(): diff --git a/session.py b/session.py index 128da7341..e1ef583cd 100644 --- a/session.py +++ b/session.py @@ -389,3 +389,64 @@ def postImage(session, attachImageFilename: str, federationList: [], if postResult: return postResult.text return None + + +def downloadImage(session, baseDir: str, url: str, + imageFilename: str, debug: bool, + force: bool = False) -> bool: + """Downloads an image + """ + if not url: + return None + + # try different image types + imageFormats = { + 'png': 'png', + 'jpg': 'jpeg', + 'jpeg': 'jpeg', + 'gif': 'gif', + 'svg': 'svg+xml', + 'webp': 'webp', + 'avif': 'avif' + } + sessionHeaders = None + for imFormat, mimeType in imageFormats.items(): + if url.endswith('.' + imFormat) or \ + '.' + imFormat + '?' in url: + sessionHeaders = { + 'Accept': 'image/' + mimeType + } + + if not sessionHeaders: + return False + + if not os.path.isfile(imageFilename) or force: + try: + if debug: + print('Downloading image url: ' + url) + result = session.get(url, + headers=sessionHeaders, + params=None) + if result.status_code < 200 or \ + result.status_code > 202: + if debug: + print('Image download failed with status ' + + str(result.status_code)) + # remove partial download + if os.path.isfile(imageFilename): + try: + os.remove(imageFilename) + except BaseException: + print('EX: downloadImage unable to delete ' + + imageFilename) + pass + else: + with open(imageFilename, 'wb') as f: + f.write(result.content) + if debug: + print('Image downloaded from ' + url) + return True + except Exception as e: + print('EX: Failed to download image: ' + + str(url) + ' ' + str(e)) + return False diff --git a/tests.py b/tests.py index 28870da73..876594409 100644 --- a/tests.py +++ b/tests.py @@ -3379,7 +3379,8 @@ def _testAddEmoji(baseDir: str): for tagName, tag in hashtags.items(): tags.append(tag) content = contentModified - contentModified = replaceEmojiFromTags(content, tags, 'content') + contentModified = \ + replaceEmojiFromTags(None, baseDir, content, tags, 'content', True) # print('contentModified: ' + contentModified) assert contentModified == '

Emoji 🍋 🍓 🍌

' diff --git a/webapp_post.py b/webapp_post.py index e05d521a7..60a67fe18 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -872,7 +872,7 @@ def _getPostTitleAnnounceHtml(baseDir: str, # add any emoji to the display name if ':' in announceDisplayName: announceDisplayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain, + addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain, announceDisplayName, False) _logPostTiming(enableTimingLog, postStartTime, '13.3.1') titleStr += \ @@ -1054,7 +1054,7 @@ def _getPostTitleReplyHtml(baseDir: str, _logPostTiming(enableTimingLog, postStartTime, '13.5') replyDisplayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, nickname, domain, + addEmojiToDisplayName(None, baseDir, httpPrefix, nickname, domain, replyDisplayName, False) _logPostTiming(enableTimingLog, postStartTime, '13.6') @@ -1316,7 +1316,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, # add any emoji to the display name if ':' in displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, displayName, False) @@ -1442,7 +1442,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if displayName: if ':' in displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, displayName, False) titleStr += \ @@ -1740,7 +1740,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if postJsonObject['object'].get('summary'): cwStr = str(postJsonObject['object']['summary']) cwStr = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, cwStr, False) contentStr += \ @@ -1768,9 +1768,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if postJsonObject['object'].get('tag') and not isPatch: contentStr = \ - replaceEmojiFromTags(contentStr, + replaceEmojiFromTags(session, baseDir, contentStr, postJsonObject['object']['tag'], - 'content') + 'content', False) if isMuted: contentStr = '' diff --git a/webapp_profile.py b/webapp_profile.py index 34d1646c2..3113b495b 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -570,12 +570,12 @@ def htmlProfile(signingPrivateKeyPem: str, if not domain: return "" displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, profileJson['name'], True) domainFull = getFullDomain(domain, port) profileDescription = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(session, baseDir, httpPrefix, nickname, domain, profileJson['summary'], False) postsButton = 'button' @@ -2271,7 +2271,7 @@ def _individualFollowAsHtml(signingPrivateKeyPem: str, avatarUrl = avatarUrl2 if displayName: displayName = \ - addEmojiToDisplayName(baseDir, httpPrefix, + addEmojiToDisplayName(None, baseDir, httpPrefix, actorNickname, domain, displayName, False) titleStr = displayName diff --git a/webapp_utils.py b/webapp_utils.py index a03c4895b..1865ff6d9 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -294,7 +294,7 @@ def updateAvatarImageCache(signingPrivateKeyPem: str, print('avatar image downloaded for ' + actor) return avatarImageFilename.replace(baseDir + '/cache', '') except Exception as e: - print('WARN: Failed to download avatar image: ' + + print('EX: Failed to download avatar image: ' + str(avatarUrl) + ' ' + str(e)) prof = 'https://www.w3.org/ns/activitystreams' if '/channel/' not in actor or '/accounts/' not in actor: @@ -781,7 +781,7 @@ def loadIndividualPostAsHtmlFromCache(baseDir: str, return postHtml -def addEmojiToDisplayName(baseDir: str, httpPrefix: str, +def addEmojiToDisplayName(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, displayName: str, inProfileName: bool) -> str: """Adds emoji icons to display names or CW on individual posts @@ -804,10 +804,14 @@ def addEmojiToDisplayName(baseDir: str, httpPrefix: str, # print('TAG: emoji tags list: ' + str(emojiTagsList)) if not inProfileName: displayName = \ - replaceEmojiFromTags(displayName, emojiTagsList, 'post header') + replaceEmojiFromTags(session, baseDir, + displayName, emojiTagsList, 'post header', + False) else: displayName = \ - replaceEmojiFromTags(displayName, emojiTagsList, 'profile') + replaceEmojiFromTags(session, baseDir, + displayName, emojiTagsList, 'profile', + False) # print('TAG: displayName after tags 2: ' + displayName) # remove any stray emoji From 775e89cd05d6f8354f06d0402ad394b2d4f3c265 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 17:23:39 +0000 Subject: [PATCH 02/20] Replacing emoji --- content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content.py b/content.py index d223e4706..e305928d9 100644 --- a/content.py +++ b/content.py @@ -300,9 +300,9 @@ def replaceEmojiFromTags(session, baseDir: str, iconName = iconName.split('.')[0] # see https://unicode.org/ # emoji/charts/full-emoji-list.html - replaced = False if '-' not in iconName: # a single code + replaced = False try: replaceChar = chr(int("0x" + iconName, 16)) content = content.replace(tagItem['name'], @@ -325,6 +325,7 @@ def replaceEmojiFromTags(session, baseDir: str, iconCodes = iconName.split('-') iconCodeSequence = '' for icode in iconCodes: + replaced = False try: iconCodeSequence += chr(int("0x" + icode, 16)) @@ -336,7 +337,6 @@ def replaceEmojiFromTags(session, baseDir: str, str(icode) + ' to chr ' + tagItem['name'] + ' ' + tagItem['icon']['url']) - break if not replaced: _saveCustomEmoji(session, baseDir, tagItem['name'], From 430ed3ce4d465507d2e934ebe5134f363d2d81ed Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 17:50:38 +0000 Subject: [PATCH 03/20] More debug --- content.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/content.py b/content.py index e305928d9..6b98efd8d 100644 --- a/content.py +++ b/content.py @@ -246,14 +246,17 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, """Saves custom emoji to file """ if not session: + if debug: + print('_saveCustomEmoji no session') return if '.' not in url: return ext = url.split('.')[-1] if ext != 'png': - print('Custom emoji is wrong format ' + url) + if debug: + print('Custom emoji is wrong format ' + url) return - emojiName = emojiName.replace(':').strip() + emojiName = emojiName.replace(':').strip().lower() customEmojiDir = baseDir + '/emojicustom' if not os.path.isdir(customEmojiDir): os.mkdir(customEmojiDir) @@ -270,6 +273,8 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, if not emojiJson.get(emojiName): emojiJson[emojiName] = emojiName saveJson(emojiJson, emojiJsonFilename) + if debug: + print('Saved custom emoji ' + emojiJsonFilename) def replaceEmojiFromTags(session, baseDir: str, From 2607ca87d47a21f0ac7a9fbfad5bc996faacb858 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 18:33:32 +0000 Subject: [PATCH 04/20] More debug --- content.py | 15 ++++++++++----- posts.py | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/content.py b/content.py index 6b98efd8d..e27c495ff 100644 --- a/content.py +++ b/content.py @@ -245,16 +245,17 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, debug: bool) -> None: """Saves custom emoji to file """ + debug = True if not session: if debug: - print('_saveCustomEmoji no session') + print('EX: _saveCustomEmoji no session') return if '.' not in url: return ext = url.split('.')[-1] if ext != 'png': if debug: - print('Custom emoji is wrong format ' + url) + print('EX: Custom emoji is wrong format ' + url) return emojiName = emojiName.replace(':').strip().lower() customEmojiDir = baseDir + '/emojicustom' @@ -263,6 +264,8 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, emojiImageFilename = customEmojiDir + '/' + emojiName + '.' + ext if not downloadImage(session, baseDir, url, emojiImageFilename, debug, False): + if debug: + print('EX: custom emoji not downloaded ' + url) return emojiJsonFilename = customEmojiDir + '/emoji.json' emojiJson = {} @@ -274,7 +277,9 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, emojiJson[emojiName] = emojiName saveJson(emojiJson, emojiJsonFilename) if debug: - print('Saved custom emoji ' + emojiJsonFilename) + print('EX: Saved custom emoji ' + emojiJsonFilename) + elif debug: + print('EX: cusom emoji already saved') def replaceEmojiFromTags(session, baseDir: str, @@ -314,7 +319,7 @@ def replaceEmojiFromTags(session, baseDir: str, replaceChar) replaced = True except BaseException: - print('EX: replaceEmojiFromTags ' + + print('EX: replaceEmojiFromTags 1 ' + 'no conversion of ' + str(iconName) + ' to chr ' + tagItem['name'] + ' ' + @@ -337,7 +342,7 @@ def replaceEmojiFromTags(session, baseDir: str, replaced = True except BaseException: iconCodeSequence = '' - print('EX: replaceEmojiFromTags ' + + print('EX: replaceEmojiFromTags 2 ' + 'no conversion of ' + str(icode) + ' to chr ' + tagItem['name'] + ' ' + diff --git a/posts.py b/posts.py index 9df4a2505..01eca5505 100644 --- a/posts.py +++ b/posts.py @@ -1344,6 +1344,7 @@ def _createPostBase(baseDir: str, tags = [] for tagName, tag in hashtagsDict.items(): tags.append(tag) + # get list of tags if nickname != 'news': content = \ From 46d171dd5c98c875f3a0a1968a0c3bdb5bc39071 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 18:38:04 +0000 Subject: [PATCH 05/20] Remove pass --- content.py | 1 - 1 file changed, 1 deletion(-) diff --git a/content.py b/content.py index e27c495ff..67f0bf2c3 100644 --- a/content.py +++ b/content.py @@ -324,7 +324,6 @@ def replaceEmojiFromTags(session, baseDir: str, str(iconName) + ' to chr ' + tagItem['name'] + ' ' + tagItem['icon']['url']) - pass if not replaced: _saveCustomEmoji(session, baseDir, tagItem['name'], From 5d56b08d1439f977c8ca973bebdd32e4c1f92b53 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 20:12:04 +0000 Subject: [PATCH 06/20] Replace argument --- content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content.py b/content.py index 67f0bf2c3..285055207 100644 --- a/content.py +++ b/content.py @@ -257,7 +257,7 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, if debug: print('EX: Custom emoji is wrong format ' + url) return - emojiName = emojiName.replace(':').strip().lower() + emojiName = emojiName.replace(':', '').strip().lower() customEmojiDir = baseDir + '/emojicustom' if not os.path.isdir(customEmojiDir): os.mkdir(customEmojiDir) From d414e204bb8a9717450f1256d8dc655ea94a0218 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 20:27:29 +0000 Subject: [PATCH 07/20] Extra debug --- content.py | 1 - session.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/content.py b/content.py index 285055207..7e274683e 100644 --- a/content.py +++ b/content.py @@ -245,7 +245,6 @@ def _saveCustomEmoji(session, baseDir: str, emojiName: str, url: str, debug: bool) -> None: """Saves custom emoji to file """ - debug = True if not session: if debug: print('EX: _saveCustomEmoji no session') diff --git a/session.py b/session.py index e1ef583cd..5f23da457 100644 --- a/session.py +++ b/session.py @@ -416,8 +416,11 @@ def downloadImage(session, baseDir: str, url: str, sessionHeaders = { 'Accept': 'image/' + mimeType } + break if not sessionHeaders: + if debug: + print('downloadImage: no session headers') return False if not os.path.isfile(imageFilename) or force: From 4bcc4930d222882a63a6b0142a718482ce3af23f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 20:31:35 +0000 Subject: [PATCH 08/20] Try custom emoji if default not found --- daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon.py b/daemon.py index 74e0fc5e2..0f50cf71e 100644 --- a/daemon.py +++ b/daemon.py @@ -6723,6 +6723,8 @@ class PubServer(BaseHTTPRequestHandler): if isImageFile(path): emojiStr = path.split('/emoji/')[1] emojiFilename = baseDir + '/emoji/' + emojiStr + if not os.path.isfile(emojiFilename): + emojiFilename = baseDir + '/emojicustom/' + emojiStr if os.path.isfile(emojiFilename): if self._etag_exists(emojiFilename): # The file has not changed From 3a31230f9c81d220420407a824be544675bcec30 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 22:45:57 +0000 Subject: [PATCH 09/20] Append custom emoji --- content.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/content.py b/content.py index 7e274683e..360920f90 100644 --- a/content.py +++ b/content.py @@ -900,6 +900,13 @@ def addHtmlTags(baseDir: str, httpPrefix: str, baseDir + '/emoji/emoji.json') emojiDict = loadJson(baseDir + '/emoji/emoji.json') + # append custom emoji to the dict + if os.path.isfile(baseDir + '/emojicustom/emoji.json'): + customEmojiDict = \ + loadJson(baseDir + '/emojicustom/emoji.json') + emojiDict = \ + dict(emojiDict.items() + customEmojiDict.items()) + # print('TAG: looking up emoji for :' + wordStr2 + ':') _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix, originalDomain, replaceEmoji, hashtags, From 570937d8f78813814e9f6a7a07acf5037c6131e6 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 22:50:26 +0000 Subject: [PATCH 10/20] Check that emoji dict was loaded --- content.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/content.py b/content.py index 360920f90..ac281212f 100644 --- a/content.py +++ b/content.py @@ -904,8 +904,9 @@ def addHtmlTags(baseDir: str, httpPrefix: str, if os.path.isfile(baseDir + '/emojicustom/emoji.json'): customEmojiDict = \ loadJson(baseDir + '/emojicustom/emoji.json') - emojiDict = \ - dict(emojiDict.items() + customEmojiDict.items()) + if customEmojiDict: + emojiDict = \ + dict(emojiDict.items() + customEmojiDict.items()) # print('TAG: looking up emoji for :' + wordStr2 + ':') _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix, From 259f4eadda87d2a234e2c3ef72e23c38e6dcbd72 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 22:56:09 +0000 Subject: [PATCH 11/20] Joining dictionaries --- content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content.py b/content.py index ac281212f..57725ecde 100644 --- a/content.py +++ b/content.py @@ -906,7 +906,7 @@ def addHtmlTags(baseDir: str, httpPrefix: str, loadJson(baseDir + '/emojicustom/emoji.json') if customEmojiDict: emojiDict = \ - dict(emojiDict.items() + customEmojiDict.items()) + dict(emojiDict, **customEmojiDict) # print('TAG: looking up emoji for :' + wordStr2 + ':') _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix, From 99d574e55a3a9729629c7b6d7084d0bae05e236f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:11:41 +0000 Subject: [PATCH 12/20] Searching for custom emoji --- webapp_search.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webapp_search.py b/webapp_search.py index a6668c9b8..f7711ea8a 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -59,6 +59,7 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, cssFilename = baseDir + '/epicyon.css' emojiLookupFilename = baseDir + '/emoji/emoji.json' + customEmojiLookupFilename = baseDir + '/emojicustom/emoji.json' # create header instanceTitle = \ @@ -77,6 +78,11 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, emojiJson = loadJson(emojiLookupFilename) if emojiJson: + customEmojiJson = None + if os.path.isfile(customEmojiLookupFilename): + customEmojiJson = loadJson(customEmojiLookupFilename) + if customEmojiJson: + emojiJson = dict(emojiJson, **customEmojiJson) results = {} for emojiName, filename in emojiJson.items(): if searchStr in emojiName: From 3d98a9f63e105585ebd4ec913d49c06ee511cdc5 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:17:14 +0000 Subject: [PATCH 13/20] Check for custom emoji during search --- webapp_search.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/webapp_search.py b/webapp_search.py index f7711ea8a..78b95e19f 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -95,15 +95,15 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, msgStr1 = translate['Copy the text then paste it into your post'] msgStr2 = ':' + '
' + msgStr1 + '
' + headingShown = True + emojiForm += \ + '

:' + emojiName + msgStr2 + filename + '"/>

' emojiForm += '' emojiForm += htmlFooter() From 3c1c2598496eb8100874b0bef23e149fd27176c7 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:26:29 +0000 Subject: [PATCH 14/20] Tidying --- webapp_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp_search.py b/webapp_search.py index 78b95e19f..84d33b4b7 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -81,7 +81,6 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, customEmojiJson = None if os.path.isfile(customEmojiLookupFilename): customEmojiJson = loadJson(customEmojiLookupFilename) - if customEmojiJson: emojiJson = dict(emojiJson, **customEmojiJson) results = {} for emojiName, filename in emojiJson.items(): From 63865bad6b0f7f3b1d10446193c06d11ce490bba Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:29:46 +0000 Subject: [PATCH 15/20] Tidying --- webapp_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp_search.py b/webapp_search.py index 84d33b4b7..6b70b5fbb 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -78,7 +78,6 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, emojiJson = loadJson(emojiLookupFilename) if emojiJson: - customEmojiJson = None if os.path.isfile(customEmojiLookupFilename): customEmojiJson = loadJson(customEmojiLookupFilename) emojiJson = dict(emojiJson, **customEmojiJson) From 9bafdef7bfde157965e244e6c9ac7db72e04803e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:34:11 +0000 Subject: [PATCH 16/20] Creating emoji dict --- content.py | 4 ++-- webapp_search.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/content.py b/content.py index 57725ecde..d01485df6 100644 --- a/content.py +++ b/content.py @@ -905,8 +905,8 @@ def addHtmlTags(baseDir: str, httpPrefix: str, customEmojiDict = \ loadJson(baseDir + '/emojicustom/emoji.json') if customEmojiDict: - emojiDict = \ - dict(emojiDict, **customEmojiDict) + newEmojiDict = dict(emojiDict, **customEmojiDict) + emojiDict = newEmojiDict # print('TAG: looking up emoji for :' + wordStr2 + ':') _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix, diff --git a/webapp_search.py b/webapp_search.py index 6b70b5fbb..68a21385f 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -80,7 +80,8 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, if emojiJson: if os.path.isfile(customEmojiLookupFilename): customEmojiJson = loadJson(customEmojiLookupFilename) - emojiJson = dict(emojiJson, **customEmojiJson) + newEmojiJson = dict(emojiJson, **customEmojiJson) + emojiDict = newEmojiJson results = {} for emojiName, filename in emojiJson.items(): if searchStr in emojiName: From 424ba859423f8bd99ba99cce31af704bc84793bc Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:39:20 +0000 Subject: [PATCH 17/20] Searching for custom emoji --- content.py | 3 +-- webapp_search.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/content.py b/content.py index d01485df6..753a08554 100644 --- a/content.py +++ b/content.py @@ -905,8 +905,7 @@ def addHtmlTags(baseDir: str, httpPrefix: str, customEmojiDict = \ loadJson(baseDir + '/emojicustom/emoji.json') if customEmojiDict: - newEmojiDict = dict(emojiDict, **customEmojiDict) - emojiDict = newEmojiDict + emojiDict = dict(emojiDict, **customEmojiDict) # print('TAG: looking up emoji for :' + wordStr2 + ':') _addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix, diff --git a/webapp_search.py b/webapp_search.py index 68a21385f..629cad276 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -71,17 +71,17 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, # does the lookup file exist? if not os.path.isfile(emojiLookupFilename): - emojiForm += '
' + \ - translate['No results'] + '
' - emojiForm += htmlFooter() - return emojiForm + if not os.path.isfile(customEmojiLookupFilename): + emojiForm += '
' + \ + translate['No results'] + '
' + emojiForm += htmlFooter() + return emojiForm emojiJson = loadJson(emojiLookupFilename) if emojiJson: if os.path.isfile(customEmojiLookupFilename): customEmojiJson = loadJson(customEmojiLookupFilename) - newEmojiJson = dict(emojiJson, **customEmojiJson) - emojiDict = newEmojiJson + emojiJson = dict(emojiJson, **customEmojiJson) results = {} for emojiName, filename in emojiJson.items(): if searchStr in emojiName: From 7a88dea6187b114784f9e168b7ef1179cc354516 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:42:37 +0000 Subject: [PATCH 18/20] Check that dictionary was loaded --- webapp_search.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/webapp_search.py b/webapp_search.py index 629cad276..b823e03d1 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -71,17 +71,17 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, # does the lookup file exist? if not os.path.isfile(emojiLookupFilename): - if not os.path.isfile(customEmojiLookupFilename): - emojiForm += '
' + \ - translate['No results'] + '
' - emojiForm += htmlFooter() - return emojiForm + emojiForm += '
' + \ + translate['No results'] + '
' + emojiForm += htmlFooter() + return emojiForm emojiJson = loadJson(emojiLookupFilename) if emojiJson: if os.path.isfile(customEmojiLookupFilename): customEmojiJson = loadJson(customEmojiLookupFilename) - emojiJson = dict(emojiJson, **customEmojiJson) + if customEmojiJson: + emojiJson = dict(emojiJson, **customEmojiJson) results = {} for emojiName, filename in emojiJson.items(): if searchStr in emojiName: From 524b7dec4c73848e35aaaad6e66ccbf4941d5f54 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:46:31 +0000 Subject: [PATCH 19/20] Show message if no emoji are found --- webapp_search.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webapp_search.py b/webapp_search.py index b823e03d1..50d8dc8b8 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -82,6 +82,7 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, customEmojiJson = loadJson(customEmojiLookupFilename) if customEmojiJson: emojiJson = dict(emojiJson, **customEmojiJson) + results = {} for emojiName, filename in emojiJson.items(): if searchStr in emojiName: @@ -89,6 +90,11 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, for emojiName, filename in emojiJson.items(): if emojiName in searchStr: results[emojiName] = filename + '.png' + + if not results: + emojiForm += '
' + \ + translate['No results'] + '
' + headingShown = False emojiForm += '
' msgStr1 = translate['Copy the text then paste it into your post'] From 1322becc98233cbdf7e1aae4b73003023093b161 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 1 Nov 2021 23:49:25 +0000 Subject: [PATCH 20/20] Directory name --- webapp_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_search.py b/webapp_search.py index 50d8dc8b8..f78f45a4a 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -101,7 +101,7 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, msgStr2 = ':