From a62641fec4e8729795244c010516125ff510b669 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 09:47:01 +0000 Subject: [PATCH 01/25] Get newswire summary from blog posts --- newswire.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/newswire.py b/newswire.py index 3f76e471b..81927fa7a 100644 --- a/newswire.py +++ b/newswire.py @@ -82,7 +82,7 @@ def getNewswireTags(text: str, maxTags: int) -> []: def addNewswireDictEntry(baseDir: str, domain: str, newswire: {}, dateStr: str, - title: str, link: str, + title: str, content: str, link: str, votesStatus: str, postFilename: str, description: str, moderated: bool, mirrored: bool, @@ -412,6 +412,7 @@ def isNewswireBlogPost(postJsonObject: {}) -> bool: return False if postJsonObject['object'].get('summary') and \ postJsonObject['object'].get('url') and \ + postJsonObject['object'].get('content') and \ postJsonObject['object'].get('published'): return isPublicPost(postJsonObject) return False @@ -443,6 +444,19 @@ def getHashtagsFromPost(postJsonObject: {}) -> []: return tags +def firstParagraph(postJsonObject: {}) -> str: + """Get the first paragraph from a blog post + to be used as a summary in the newswire feed + """ + content = postJsonObject['object']['content'] + if '

' not in content or '

' not in content: + return removeHtml(content) + paragraph = content.split('

')[1] + if '

' in paragraph: + paragraph = paragraph.split('

')[0] + return removeHtml(paragraph) + + def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, newswire: {}, maxBlogsPerAccount: int, @@ -500,7 +514,7 @@ def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, votes = [] if os.path.isfile(fullPostFilename + '.votes'): votes = loadJson(fullPostFilename + '.votes') - description = '' + description = firstParagraph(postJsonObject) addNewswireDictEntry(baseDir, domain, newswire, published, postJsonObject['object']['summary'], From bb936c455af7c79578874d3db1db711d0336086f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 10:05:28 +0000 Subject: [PATCH 02/25] Remove extraneous argument --- newswire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newswire.py b/newswire.py index 81927fa7a..f69d5350d 100644 --- a/newswire.py +++ b/newswire.py @@ -82,7 +82,7 @@ def getNewswireTags(text: str, maxTags: int) -> []: def addNewswireDictEntry(baseDir: str, domain: str, newswire: {}, dateStr: str, - title: str, content: str, link: str, + title: str, link: str, votesStatus: str, postFilename: str, description: str, moderated: bool, mirrored: bool, From 49414c70128629c62db4d4d07e83d07b7a146aae Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 10:30:25 +0000 Subject: [PATCH 03/25] Include description within rss feeds --- blog.py | 6 ++++++ newswire.py | 14 +------------- utils.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/blog.py b/blog.py index 4c3e83c70..606fc923f 100644 --- a/blog.py +++ b/blog.py @@ -19,6 +19,7 @@ from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost from utils import loadJson +from utils import firstParagraph from posts import createBlogsTimeline from newswire import rss2Header from newswire import rss2Footer @@ -312,9 +313,12 @@ def htmlBlogPostRSS2(authorized: bool, pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") titleStr = postJsonObject['object']['summary'] rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") + description = firstParagraph(postJsonObject) rssStr = ' ' rssStr += ' ' + titleStr + '' rssStr += ' ' + messageLink + '' + rssStr += \ + ' ' + description + '' rssStr += ' ' + rssDateStr + '' rssStr += ' ' return rssStr @@ -339,8 +343,10 @@ def htmlBlogPostRSS3(authorized: bool, pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") titleStr = postJsonObject['object']['summary'] rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") + description = firstParagraph(postJsonObject) rssStr = 'title: ' + titleStr + '\n' rssStr += 'link: ' + messageLink + '\n' + rssStr += 'description: ' + description + '\n' rssStr += 'created: ' + rssDateStr + '\n\n' return rssStr diff --git a/newswire.py b/newswire.py index f69d5350d..e738fef4e 100644 --- a/newswire.py +++ b/newswire.py @@ -12,6 +12,7 @@ from socket import error as SocketError import errno from datetime import datetime from collections import OrderedDict +from utils import firstParagraph from utils import isPublicPost from utils import locatePost from utils import loadJson @@ -444,19 +445,6 @@ def getHashtagsFromPost(postJsonObject: {}) -> []: return tags -def firstParagraph(postJsonObject: {}) -> str: - """Get the first paragraph from a blog post - to be used as a summary in the newswire feed - """ - content = postJsonObject['object']['content'] - if '

' not in content or '

' not in content: - return removeHtml(content) - paragraph = content.split('

')[1] - if '

' in paragraph: - paragraph = paragraph.split('

')[0] - return removeHtml(paragraph) - - def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, newswire: {}, maxBlogsPerAccount: int, diff --git a/utils.py b/utils.py index 08d21f159..6640e1dde 100644 --- a/utils.py +++ b/utils.py @@ -19,6 +19,19 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar +def firstParagraph(postJsonObject: {}) -> str: + """Get the first paragraph from a blog post + to be used as a summary in the newswire feed + """ + content = postJsonObject['object']['content'] + if '

' not in content or '

' not in content: + return removeHtml(content) + paragraph = content.split('

')[1] + if '

' in paragraph: + paragraph = paragraph.split('

')[0] + return removeHtml(paragraph) + + def removeHtml(content: str) -> str: """Removes html links from the given content. Used to ensure that profile descriptions don't contain dubious content From 30e63970f63ef4b84ff3ac558f20d9333adb1980 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 10:45:33 +0000 Subject: [PATCH 04/25] Include description in newswire rss feed --- blog.py | 8 +++++--- newswire.py | 7 +++++-- utils.py | 3 +-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/blog.py b/blog.py index 606fc923f..96d52a1fc 100644 --- a/blog.py +++ b/blog.py @@ -19,7 +19,7 @@ from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost from utils import loadJson -from utils import firstParagraph +from utils import firstParagraphFromString from posts import createBlogsTimeline from newswire import rss2Header from newswire import rss2Footer @@ -313,7 +313,8 @@ def htmlBlogPostRSS2(authorized: bool, pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") titleStr = postJsonObject['object']['summary'] rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") - description = firstParagraph(postJsonObject) + content = postJsonObject['object']['content'] + description = firstParagraphFromString(content) rssStr = ' ' rssStr += ' ' + titleStr + '' rssStr += ' ' + messageLink + '' @@ -343,7 +344,8 @@ def htmlBlogPostRSS3(authorized: bool, pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") titleStr = postJsonObject['object']['summary'] rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") - description = firstParagraph(postJsonObject) + content = postJsonObject['object']['content'] + description = firstParagraphFromString(content) rssStr = 'title: ' + titleStr + '\n' rssStr += 'link: ' + messageLink + '\n' rssStr += 'description: ' + description + '\n' diff --git a/newswire.py b/newswire.py index e738fef4e..4fca15a78 100644 --- a/newswire.py +++ b/newswire.py @@ -12,7 +12,7 @@ from socket import error as SocketError import errno from datetime import datetime from collections import OrderedDict -from utils import firstParagraph +from utils import firstParagraphFromString from utils import isPublicPost from utils import locatePost from utils import loadJson @@ -387,6 +387,8 @@ def getRSSfromDict(baseDir: str, newswire: {}, continue rssStr += '\n' rssStr += ' ' + fields[0] + '\n' + description = firstParagraphFromString(fields[4]) + rssStr += ' ' + description + '\n' url = fields[1] if domainFull not in url: url = httpPrefix + '://' + domainFull + url @@ -502,7 +504,8 @@ def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, votes = [] if os.path.isfile(fullPostFilename + '.votes'): votes = loadJson(fullPostFilename + '.votes') - description = firstParagraph(postJsonObject) + content = postJsonObject['object']['content'] + description = firstParagraphFromString(content) addNewswireDictEntry(baseDir, domain, newswire, published, postJsonObject['object']['summary'], diff --git a/utils.py b/utils.py index 6640e1dde..11d00aa03 100644 --- a/utils.py +++ b/utils.py @@ -19,11 +19,10 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar -def firstParagraph(postJsonObject: {}) -> str: +def firstParagraphFromString(content: str) -> str: """Get the first paragraph from a blog post to be used as a summary in the newswire feed """ - content = postJsonObject['object']['content'] if '

' not in content or '

' not in content: return removeHtml(content) paragraph = content.split('

')[1] From 2b15de6ee79953ff611b2a309f816e885db51d7e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 10:52:07 +0000 Subject: [PATCH 05/25] Function sequence --- utils.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/utils.py b/utils.py index 11d00aa03..b653d280c 100644 --- a/utils.py +++ b/utils.py @@ -19,18 +19,6 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar -def firstParagraphFromString(content: str) -> str: - """Get the first paragraph from a blog post - to be used as a summary in the newswire feed - """ - if '

' not in content or '

' not in content: - return removeHtml(content) - paragraph = content.split('

')[1] - if '

' in paragraph: - paragraph = paragraph.split('

')[0] - return removeHtml(paragraph) - - def removeHtml(content: str) -> str: """Removes html links from the given content. Used to ensure that profile descriptions don't contain dubious content @@ -50,6 +38,18 @@ def removeHtml(content: str) -> str: return result +def firstParagraphFromString(content: str) -> str: + """Get the first paragraph from a blog post + to be used as a summary in the newswire feed + """ + if '

' not in content or '

' not in content: + return removeHtml(content) + paragraph = content.split('

')[1] + if '

' in paragraph: + paragraph = paragraph.split('

')[0] + return removeHtml(paragraph) + + def isSystemAccount(nickname: str) -> bool: """Returns true if the given nickname is a system account """ From 2855af60254662ed5cb1358aad69ccfcddd69101 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 10:54:29 +0000 Subject: [PATCH 06/25] Extra removal --- newsdaemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newsdaemon.py b/newsdaemon.py index 695eaef71..ac3be97db 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -521,6 +521,7 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, if rssDescription.startswith('', '') + rssDescription = rssDescription.replace(']]', '') if '&' in rssDescription: rssDescription = html.unescape(rssDescription) rssDescription = '

' + rssDescription + '

' From db390b1a2325f2042351b472f5c832fbb192d059 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 11:04:52 +0000 Subject: [PATCH 07/25] Extra check for full url --- newswire.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/newswire.py b/newswire.py index 4fca15a78..7874ecb9a 100644 --- a/newswire.py +++ b/newswire.py @@ -390,8 +390,9 @@ def getRSSfromDict(baseDir: str, newswire: {}, description = firstParagraphFromString(fields[4]) rssStr += ' ' + description + '\n' url = fields[1] - if domainFull not in url: - url = httpPrefix + '://' + domainFull + url + if '://' not in url: + if domainFull not in url: + url = httpPrefix + '://' + domainFull + url rssStr += ' ' + url + '\n' rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") From 9d89bb5f94dd91e05ac10ef91b73f7450c6e1110 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 11:24:43 +0000 Subject: [PATCH 08/25] Test for getting the first paragraph --- tests.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests.py b/tests.py index 8a75dbedc..f835378c0 100644 --- a/tests.py +++ b/tests.py @@ -32,6 +32,7 @@ from follow import clearFollows from follow import clearFollowers from follow import sendFollowRequestViaServer from follow import sendUnfollowRequestViaServer +from utils import firstParagraphFromString from utils import removeIdEnding from utils import siteIsActive from utils import updateRecentPostsCache @@ -2336,8 +2337,22 @@ def testGetNewswireTags(): assert '#ExcitingHashtag' in tags +def testFirstParagraphFromString(): + print('testFirstParagraphFromString') + testStr = \ + '

This is a test

' + \ + '

This is another paragraph

' + resultStr = firstParagraphFromString(testStr) + assert resultStr == 'This is a test' + + testStr = 'Testing without html' + resultStr = firstParagraphFromString(testStr) + assert resultStr == testStr + + def runAllTests(): print('Running tests...') + testFirstParagraphFromString() testGetNewswireTags() testHashtagRuleTree() testRemoveHtmlTag() From 4dc132c7bc0e466f60d067eeb920a9c81253f97d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 12:40:40 +0000 Subject: [PATCH 09/25] Get first paragraph for feed description --- webinterface.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/webinterface.py b/webinterface.py index a2933611d..f1d27bac5 100644 --- a/webinterface.py +++ b/webinterface.py @@ -25,6 +25,7 @@ from ssb import getSSBAddress from tox import getToxAddress from matrix import getMatrixAddress from donate import getDonationUrl +from utils import firstParagraphFromString from utils import getCSS from utils import isSystemAccount from utils import removeIdEnding @@ -969,10 +970,11 @@ def rssHashtagSearch(nickname: str, domain: str, port: int, ' ' + \ postJsonObject['object']['summary'] + \ '' + description = postJsonObject['object']['content'] + description = firstParagraphFromString(description) hashtagFeed += \ ' ' + description + ']]>' hashtagFeed += \ ' ' + rssDateStr + '' if postJsonObject['object'].get('attachment'): From 87437fe55c3e9130ac377acaf7fec76ad9e98466 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 12:42:33 +0000 Subject: [PATCH 10/25] Don't need cdata --- webinterface.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webinterface.py b/webinterface.py index f1d27bac5..0ce2d12a8 100644 --- a/webinterface.py +++ b/webinterface.py @@ -973,8 +973,7 @@ def rssHashtagSearch(nickname: str, domain: str, port: int, description = postJsonObject['object']['content'] description = firstParagraphFromString(description) hashtagFeed += \ - ' ' + ' ' + description + '' hashtagFeed += \ ' ' + rssDateStr + '' if postJsonObject['object'].get('attachment'): From 1b69ea8bda18613fff38c2481e8b3b7debe34a72 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 13:01:01 +0000 Subject: [PATCH 11/25] Article html tag on blog --- blog.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/blog.py b/blog.py index 96d52a1fc..19e572bcb 100644 --- a/blog.py +++ b/blog.py @@ -166,10 +166,12 @@ def htmlBlogPostContent(authorized: bool, if postJsonObject['object'].get('id'): messageLink = postJsonObject['object']['id'].replace('/statuses/', '/') titleStr = '' + articleAdded = False if postJsonObject['object'].get('summary'): titleStr = postJsonObject['object']['summary'] - blogStr += '

' + \ + blogStr += '

' + \ titleStr + '

\n' + articleAdded = True # get the handle of the author if postJsonObject['object'].get('attributedTo'): @@ -232,7 +234,10 @@ def htmlBlogPostContent(authorized: bool, contentStr = replaceEmojiFromTags(contentStr, postJsonObject['object']['tag'], 'content') - blogStr += '
' + contentStr + '\n' + if articleAdded: + blogStr += '
' + contentStr + '
\n' + else: + blogStr += '
' + contentStr + '
\n' citationsStr = '' if postJsonObject['object'].get('tag'): From 46b37295064d95fc68e5525748b43386287d50fb Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 13:54:47 +0000 Subject: [PATCH 12/25] Differing titles for left column rss feed --- translations/ar.json | 3 ++- translations/ca.json | 3 ++- translations/cy.json | 3 ++- translations/de.json | 3 ++- translations/en.json | 3 ++- translations/es.json | 3 ++- translations/fr.json | 3 ++- translations/ga.json | 3 ++- translations/hi.json | 3 ++- translations/it.json | 3 ++- translations/ja.json | 3 ++- translations/oc.json | 3 ++- translations/pt.json | 3 ++- translations/ru.json | 3 ++- translations/zh.json | 3 ++- webinterface.py | 19 ++++++++++++------- 16 files changed, 42 insertions(+), 22 deletions(-) diff --git a/translations/ar.json b/translations/ar.json index 1118a56b1..f39a708a8 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -327,5 +327,6 @@ "Create an article": "قم بإنشاء مقال", "Settings": "إعدادات", "Citations": "اقتباسات", - "Choose newswire items referenced in your article": "اختر العناصر الإخبارية المشار إليها في مقالتك" + "Choose newswire items referenced in your article": "اختر العناصر الإخبارية المشار إليها في مقالتك", + "RSS feed for your blog": "تغذية RSS لمدونتك" } diff --git a/translations/ca.json b/translations/ca.json index 797ad5469..29c09497c 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -327,5 +327,6 @@ "Create an article": "Creeu un article", "Settings": "Configuració", "Citations": "Cites", - "Choose newswire items referenced in your article": "Trieu articles de newswire als quals faci referència el vostre article" + "Choose newswire items referenced in your article": "Trieu articles de newswire als quals faci referència el vostre article", + "RSS feed for your blog": "Feed RSS del vostre bloc" } diff --git a/translations/cy.json b/translations/cy.json index d5cdb532d..d6791b73b 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -327,5 +327,6 @@ "Create an article": "Creu erthygl", "Settings": "Gosodiadau", "Citations": "Dyfyniadau", - "Choose newswire items referenced in your article": "Dewiswch eitemau newyddion y cyfeirir atynt yn eich erthygl" + "Choose newswire items referenced in your article": "Dewiswch eitemau newyddion y cyfeirir atynt yn eich erthygl", + "RSS feed for your blog": "Porthiant RSS ar gyfer eich blog" } diff --git a/translations/de.json b/translations/de.json index b8eb64df8..b9a4fd7ec 100644 --- a/translations/de.json +++ b/translations/de.json @@ -327,5 +327,6 @@ "Create an article": "Erstellen Sie einen Artikel", "Settings": "Einstellungen", "Citations": "Zitate", - "Choose newswire items referenced in your article": "Wählen Sie Newswire-Artikel aus, auf die in Ihrem Artikel verwiesen wird" + "Choose newswire items referenced in your article": "Wählen Sie Newswire-Artikel aus, auf die in Ihrem Artikel verwiesen wird", + "RSS feed for your blog": "RSS-Feed für Ihr Blog" } diff --git a/translations/en.json b/translations/en.json index 0b7b9b3c3..f2c6a566d 100644 --- a/translations/en.json +++ b/translations/en.json @@ -327,5 +327,6 @@ "Create an article": "Create an article", "Settings": "Settings", "Citations": "Citations", - "Choose newswire items referenced in your article": "Choose newswire items referenced in your article" + "Choose newswire items referenced in your article": "Choose newswire items referenced in your article", + "RSS feed for your blog": "RSS feed for your blog" } diff --git a/translations/es.json b/translations/es.json index dac48137c..693590fb6 100644 --- a/translations/es.json +++ b/translations/es.json @@ -327,5 +327,6 @@ "Create an article": "Crea un articulo", "Settings": "Configuraciones", "Citations": "Citas", - "Choose newswire items referenced in your article": "Elija elementos de Newswire a los que se hace referencia en su artículo" + "Choose newswire items referenced in your article": "Elija elementos de Newswire a los que se hace referencia en su artículo", + "RSS feed for your blog": "Fuente RSS para tu blog" } diff --git a/translations/fr.json b/translations/fr.json index e1edeefb3..f5422ddbb 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -327,5 +327,6 @@ "Create an article": "Créer un article", "Settings": "Réglages", "Citations": "Citations", - "Choose newswire items referenced in your article": "Choisissez les éléments de fil d'actualité référencés dans votre article" + "Choose newswire items referenced in your article": "Choisissez les éléments de fil d'actualité référencés dans votre article", + "RSS feed for your blog": "Flux RSS pour votre blog" } diff --git a/translations/ga.json b/translations/ga.json index ba28b61e6..3836db30d 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -327,5 +327,6 @@ "Create an article": "Cruthaigh alt", "Settings": "Socruithe", "Citations": "Citations", - "Choose newswire items referenced in your article": "Roghnaigh míreanna sreanga nuachta dá dtagraítear i d’alt" + "Choose newswire items referenced in your article": "Roghnaigh míreanna sreanga nuachta dá dtagraítear i d’alt", + "RSS feed for your blog": "Fotha RSS do do bhlag" } diff --git a/translations/hi.json b/translations/hi.json index 11daf17df..31b486e7f 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -327,5 +327,6 @@ "Create an article": "एक लेख बनाएँ", "Settings": "समायोजन", "Citations": "उद्धरण", - "Choose newswire items referenced in your article": "अपने लेख में संदर्भित newswire आइटम चुनें" + "Choose newswire items referenced in your article": "अपने लेख में संदर्भित newswire आइटम चुनें", + "RSS feed for your blog": "RSS आपके ब्लॉग के लिए फ़ीड करता है" } diff --git a/translations/it.json b/translations/it.json index 2470fc5d9..f65e18c98 100644 --- a/translations/it.json +++ b/translations/it.json @@ -327,5 +327,6 @@ "Create an article": "Crea un articolo", "Settings": "impostazioni", "Citations": "Citazioni", - "Choose newswire items referenced in your article": "Scegli gli articoli del newswire a cui fa riferimento il tuo articolo" + "Choose newswire items referenced in your article": "Scegli gli articoli del newswire a cui fa riferimento il tuo articolo", + "RSS feed for your blog": "Feed RSS per il tuo blog" } diff --git a/translations/ja.json b/translations/ja.json index d6b71d6ca..bafa38784 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -327,5 +327,6 @@ "Create an article": "記事を作成する", "Settings": "設定", "Citations": "引用", - "Choose newswire items referenced in your article": "あなたの記事で参照されているニュースワイヤーアイテムを選択してください" + "Choose newswire items referenced in your article": "あなたの記事で参照されているニュースワイヤーアイテムを選択してください", + "RSS feed for your blog": "ブログのRSSフィード" } diff --git a/translations/oc.json b/translations/oc.json index efda869e1..250302075 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -323,5 +323,6 @@ "Create an article": "Create an article", "Settings": "Settings", "Citations": "Citations", - "Choose newswire items referenced in your article": "Choose newswire items referenced in your article" + "Choose newswire items referenced in your article": "Choose newswire items referenced in your article", + "RSS feed for your blog": "RSS feed for your blog" } diff --git a/translations/pt.json b/translations/pt.json index 316fdcf85..831c1d106 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -327,5 +327,6 @@ "Create an article": "Crie um artigo", "Settings": "Definições", "Citations": "Citações", - "Choose newswire items referenced in your article": "Escolha os itens de notícias mencionados em seu artigo" + "Choose newswire items referenced in your article": "Escolha os itens de notícias mencionados em seu artigo", + "RSS feed for your blog": "Feed RSS para o seu blog" } diff --git a/translations/ru.json b/translations/ru.json index 885238420..2665feb49 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -327,5 +327,6 @@ "Create an article": "Создать статью", "Settings": "Настройки", "Citations": "Цитаты", - "Choose newswire items referenced in your article": "Выберите элементы ленты новостей, на которые есть ссылки в вашей статье" + "Choose newswire items referenced in your article": "Выберите элементы ленты новостей, на которые есть ссылки в вашей статье", + "RSS feed for your blog": "RSS-канал для вашего блога" } diff --git a/translations/zh.json b/translations/zh.json index f7ba586ec..635ceeeff 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -327,5 +327,6 @@ "Create an article": "建立文章", "Settings": "设定值", "Citations": "引文", - "Choose newswire items referenced in your article": "选择文章中引用的新闻专栏文章" + "Choose newswire items referenced in your article": "选择文章中引用的新闻专栏文章", + "RSS feed for your blog": "您博客的RSS供稿" } diff --git a/webinterface.py b/webinterface.py index 0ce2d12a8..8f88455ce 100644 --- a/webinterface.py +++ b/webinterface.py @@ -3642,7 +3642,8 @@ def htmlProfile(rssIconAtTop: bool, getLeftColumnContent(baseDir, 'news', domainFull, httpPrefix, translate, iconsDir, False, - False, None, rssIconAtTop, True) + False, None, rssIconAtTop, True, + True) profileHeaderStr += ' \n' profileHeaderStr += ' \n' else: @@ -5681,7 +5682,8 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, httpPrefix: str, translate: {}, iconsDir: str, editor: bool, showBackButton: bool, timelinePath: str, - rssIconAtTop: bool, showHeaderImage: bool) -> str: + rssIconAtTop: bool, showHeaderImage: bool, + frontPage: bool) -> str: """Returns html content for the left column """ htmlStr = '' @@ -5756,12 +5758,15 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, else: # rss feed for all accounts on the instance rssUrl = httpPrefix + '://' + domainFull + '/blog/rss.xml' + if not frontPage: + rssTitle = translate['RSS feed for your blog'] + else: + rssTitle = translate['RSS feed for this site'] rssIconStr = \ ' ' + \ '' + \ - translate['RSS feed for this site'] + \ - '\n' if rssIconAtTop: htmlStr += rssIconStr @@ -6234,7 +6239,7 @@ def htmlLinksMobile(cssCache: {}, baseDir: str, httpPrefix, translate, iconsDir, editor, False, timelinePath, - rssIconAtTop, False) + rssIconAtTop, False, False) htmlStr += '\n' + htmlFooter() return htmlStr @@ -7124,7 +7129,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, getLeftColumnContent(baseDir, nickname, domainFull, httpPrefix, translate, iconsDir, editor, False, None, rssIconAtTop, - True) + True, False) tlStr += ' ' + \ leftColumnStr + ' \n' # center column containing posts From 818b7237c715956942f5708d58fb50ff681ded9a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 15:46:48 +0000 Subject: [PATCH 13/25] Update rss description after editing news post --- daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 10f5e76fa..cdac5b1f4 100644 --- a/daemon.py +++ b/daemon.py @@ -166,6 +166,7 @@ from shares import getSharesFeedForPerson from shares import addShare from shares import removeShare from shares import expireShares +from utils import firstParagraphFromString from utils import clearFromPostCaches from utils import containsInvalidChars from utils import isSystemAccount @@ -3272,7 +3273,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire[publishedDate][0] = \ newsPostTitle self.server.newswire[publishedDate][4] = \ - newsPostContent + firstParagraphFromString(newsPostContent) # save newswire newswireStateFilename = \ baseDir + '/accounts/.newswirestate.json' From 9c1c790fd5e065d4b2ee67b3985ef3a74f2d91af Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 16:50:50 +0000 Subject: [PATCH 14/25] Newswire links directly to sources --- newsdaemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newsdaemon.py b/newsdaemon.py index ac3be97db..53d47369c 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -664,8 +664,8 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, os.remove(filename + '.arrived') # set the url - newswire[originalDateStr][1] = \ - '/users/news/statuses/' + statusNumber + # newswire[originalDateStr][1] = \ + # '/users/news/statuses/' + statusNumber # set the filename newswire[originalDateStr][3] = filename From d6b22535f75a5b334c5a988c988982d16b722af3 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 16:52:57 +0000 Subject: [PATCH 15/25] Comments --- newsdaemon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/newsdaemon.py b/newsdaemon.py index 53d47369c..0e27bb4c1 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -663,9 +663,11 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, if os.path.isfile(filename + '.arrived'): os.remove(filename + '.arrived') - # set the url + # setting the url here links to the activitypub object + # stored locally # newswire[originalDateStr][1] = \ # '/users/news/statuses/' + statusNumber + # set the filename newswire[originalDateStr][3] = filename From 5c42d47f55d2b1f9727cf75a2a4e019d5a84fe41 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 8 Nov 2020 18:29:01 +0000 Subject: [PATCH 16/25] Newswire links directly to sources --- newsdaemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newsdaemon.py b/newsdaemon.py index 0e27bb4c1..7b9b43430 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -504,8 +504,8 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, if os.path.isfile(filename): # don't create the post if it already exists # set the url - newswire[originalDateStr][1] = \ - '/users/news/statuses/' + statusNumber + # newswire[originalDateStr][1] = \ + # '/users/news/statuses/' + statusNumber # set the filename newswire[originalDateStr][3] = filename continue From ddc63da75566717708106c41297c61ed44591e35 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 9 Nov 2020 13:18:28 +0000 Subject: [PATCH 17/25] Rename module --- blog.py | 10 ++-- daemon.py | 98 ++++++++++++++++++------------------ inbox.py | 6 +-- webinterface.py => webapp.py | 0 4 files changed, 57 insertions(+), 57 deletions(-) rename webinterface.py => webapp.py (100%) diff --git a/blog.py b/blog.py index 19e572bcb..2f0d4e70a 100644 --- a/blog.py +++ b/blog.py @@ -10,11 +10,11 @@ import os from datetime import datetime from content import replaceEmojiFromTags -from webinterface import getIconsDir -from webinterface import getPostAttachmentsAsHtml -from webinterface import htmlHeader -from webinterface import htmlFooter -from webinterface import addEmbeddedElements +from webapp import getIconsDir +from webapp import getPostAttachmentsAsHtml +from webapp import htmlHeader +from webapp import htmlFooter +from webapp import addEmbeddedElements from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost diff --git a/daemon.py b/daemon.py index cdac5b1f4..a58e30875 100644 --- a/daemon.py +++ b/daemon.py @@ -113,55 +113,55 @@ from blog import htmlBlogView from blog import htmlBlogPage from blog import htmlBlogPost from blog import htmlEditBlog -from webinterface import htmlCitations -from webinterface import htmlFollowingList -from webinterface import getBlogAddress -from webinterface import setBlogAddress -from webinterface import htmlCalendarDeleteConfirm -from webinterface import htmlDeletePost -from webinterface import htmlAbout -from webinterface import htmlRemoveSharedItem -from webinterface import htmlInboxDMs -from webinterface import htmlInboxReplies -from webinterface import htmlInboxMedia -from webinterface import htmlInboxBlogs -from webinterface import htmlInboxNews -from webinterface import htmlUnblockConfirm -from webinterface import htmlPersonOptions -from webinterface import htmlIndividualPost -from webinterface import htmlProfile -from webinterface import htmlInbox -from webinterface import htmlBookmarks -from webinterface import htmlEvents -from webinterface import htmlShares -from webinterface import htmlOutbox -from webinterface import htmlModeration -from webinterface import htmlPostReplies -from webinterface import htmlLogin -from webinterface import htmlSuspended -from webinterface import htmlGetLoginCredentials -from webinterface import htmlNewPost -from webinterface import htmlFollowConfirm -from webinterface import htmlCalendar -from webinterface import htmlSearch -from webinterface import htmlNewswireMobile -from webinterface import htmlLinksMobile -from webinterface import htmlSearchEmoji -from webinterface import htmlSearchEmojiTextEntry -from webinterface import htmlUnfollowConfirm -from webinterface import htmlProfileAfterSearch -from webinterface import htmlEditProfile -from webinterface import htmlEditLinks -from webinterface import htmlEditNewswire -from webinterface import htmlEditNewsPost -from webinterface import htmlTermsOfService -from webinterface import htmlSkillsSearch -from webinterface import htmlHistorySearch -from webinterface import htmlHashtagSearch -from webinterface import rssHashtagSearch -from webinterface import htmlModerationInfo -from webinterface import htmlSearchSharedItems -from webinterface import htmlHashtagBlocked +from webapp import htmlCitations +from webapp import htmlFollowingList +from webapp import getBlogAddress +from webapp import setBlogAddress +from webapp import htmlCalendarDeleteConfirm +from webapp import htmlDeletePost +from webapp import htmlAbout +from webapp import htmlRemoveSharedItem +from webapp import htmlInboxDMs +from webapp import htmlInboxReplies +from webapp import htmlInboxMedia +from webapp import htmlInboxBlogs +from webapp import htmlInboxNews +from webapp import htmlUnblockConfirm +from webapp import htmlPersonOptions +from webapp import htmlIndividualPost +from webapp import htmlProfile +from webapp import htmlInbox +from webapp import htmlBookmarks +from webapp import htmlEvents +from webapp import htmlShares +from webapp import htmlOutbox +from webapp import htmlModeration +from webapp import htmlPostReplies +from webapp import htmlLogin +from webapp import htmlSuspended +from webapp import htmlGetLoginCredentials +from webapp import htmlNewPost +from webapp import htmlFollowConfirm +from webapp import htmlCalendar +from webapp import htmlSearch +from webapp import htmlNewswireMobile +from webapp import htmlLinksMobile +from webapp import htmlSearchEmoji +from webapp import htmlSearchEmojiTextEntry +from webapp import htmlUnfollowConfirm +from webapp import htmlProfileAfterSearch +from webapp import htmlEditProfile +from webapp import htmlEditLinks +from webapp import htmlEditNewswire +from webapp import htmlEditNewsPost +from webapp import htmlTermsOfService +from webapp import htmlSkillsSearch +from webapp import htmlHistorySearch +from webapp import htmlHashtagSearch +from webapp import rssHashtagSearch +from webapp import htmlModerationInfo +from webapp import htmlSearchSharedItems +from webapp import htmlHashtagBlocked from shares import getSharesFeedForPerson from shares import addShare from shares import removeShare diff --git a/inbox.py b/inbox.py index a15b9f9c8..4916af2fa 100644 --- a/inbox.py +++ b/inbox.py @@ -56,9 +56,9 @@ from posts import isMuted from posts import isImageMedia from posts import sendSignedJson from posts import sendToFollowersThread -from webinterface import individualPostAsHtml -from webinterface import getIconsDir -from webinterface import removeOldHashtags +from webapp import individualPostAsHtml +from webapp import getIconsDir +from webapp import removeOldHashtags from question import questionUpdateVotes from media import replaceYouTube from git import isGitPatch diff --git a/webinterface.py b/webapp.py similarity index 100% rename from webinterface.py rename to webapp.py From 3b1d77962aaf8f2e1c8fad383a9e7eb756132221 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 9 Nov 2020 15:22:59 +0000 Subject: [PATCH 18/25] Migrating functions from webapp --- daemon.py | 2 +- feeds.py | 23 +++ webapp.py | 386 ++--------------------------------------------- webapp_utils.py | 391 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 374 deletions(-) create mode 100644 feeds.py create mode 100644 webapp_utils.py diff --git a/daemon.py b/daemon.py index a58e30875..967aac905 100644 --- a/daemon.py +++ b/daemon.py @@ -113,10 +113,10 @@ from blog import htmlBlogView from blog import htmlBlogPage from blog import htmlBlogPost from blog import htmlEditBlog +from webapp_utils import setBlogAddress from webapp import htmlCitations from webapp import htmlFollowingList from webapp import getBlogAddress -from webapp import setBlogAddress from webapp import htmlCalendarDeleteConfirm from webapp import htmlDeletePost from webapp import htmlAbout diff --git a/feeds.py b/feeds.py new file mode 100644 index 000000000..a3a77ca5c --- /dev/null +++ b/feeds.py @@ -0,0 +1,23 @@ +__filename__ = "feeds.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.1.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + + +def rss2TagHeader(hashtag: str, httpPrefix: str, domainFull: str) -> str: + rssStr = "" + rssStr += "" + rssStr += '' + rssStr += ' #' + hashtag + '' + rssStr += ' ' + httpPrefix + '://' + domainFull + \ + '/tags/rss2/' + hashtag + '' + return rssStr + + +def rss2TagFooter() -> str: + rssStr = '' + rssStr += '' + return rssStr diff --git a/webapp.py b/webapp.py index 8f88455ce..a3a9be795 100644 --- a/webapp.py +++ b/webapp.py @@ -1,4 +1,4 @@ -__filename__ = "webinterface.py" +__filename__ = "webapp.py" __author__ = "Bob Mottram" __license__ = "AGPL3+" __version__ = "1.1.0" @@ -9,7 +9,6 @@ __status__ = "Production" import time import os import urllib.parse -from collections import OrderedDict from datetime import datetime from datetime import date from dateutil.parser import parse @@ -76,7 +75,6 @@ from content import replaceEmojiFromTags from content import removeLongWords from skills import getSkills from cache import getPersonFromCache -from cache import storePersonInCache from shares import getValidSharedItemID from happening import todaysEventsCheck from happening import thisWeeksEventsCheck @@ -87,246 +85,18 @@ from theme import getThemesList from petnames import getPetName from followingCalendar import receivingCalendarEvents from devices import E2EEdecryptMessageFromDevice - - -def getAltPath(actor: str, domainFull: str, callingDomain: str) -> str: - """Returns alternate path from the actor - eg. https://clearnetdomain/path becomes http://oniondomain/path - """ - postActor = actor - if callingDomain not in actor and domainFull in actor: - if callingDomain.endswith('.onion') or \ - callingDomain.endswith('.i2p'): - postActor = \ - 'http://' + callingDomain + actor.split(domainFull)[1] - print('Changed POST domain from ' + actor + ' to ' + postActor) - return postActor - - -def getContentWarningButton(postID: str, translate: {}, - content: str) -> str: - """Returns the markup for a content warning button - """ - return '
' + \ - translate['SHOW MORE'] + '' + \ - '
' + content + \ - '
\n' - - -def getBlogAddress(actorJson: {}) -> str: - """Returns blog address for the given actor - """ - if not actorJson.get('attachment'): - return '' - for propertyValue in actorJson['attachment']: - if not propertyValue.get('name'): - continue - if not propertyValue['name'].lower().startswith('blog'): - continue - if not propertyValue.get('type'): - continue - if not propertyValue.get('value'): - continue - if propertyValue['type'] != 'PropertyValue': - continue - propertyValue['value'] = propertyValue['value'].strip() - prefixes = getProtocolPrefixes() - prefixFound = False - for prefix in prefixes: - if propertyValue['value'].startswith(prefix): - prefixFound = True - break - if not prefixFound: - continue - if '.' not in propertyValue['value']: - continue - if ' ' in propertyValue['value']: - continue - if ',' in propertyValue['value']: - continue - return propertyValue['value'] - return '' - - -def setBlogAddress(actorJson: {}, blogAddress: str) -> None: - """Sets an blog address for the given actor - """ - if not actorJson.get('attachment'): - actorJson['attachment'] = [] - - # remove any existing value - propertyFound = None - for propertyValue in actorJson['attachment']: - if not propertyValue.get('name'): - continue - if not propertyValue.get('type'): - continue - if not propertyValue['name'].lower().startswith('blog'): - continue - propertyFound = propertyValue - break - if propertyFound: - actorJson['attachment'].remove(propertyFound) - - prefixes = getProtocolPrefixes() - prefixFound = False - for prefix in prefixes: - if blogAddress.startswith(prefix): - prefixFound = True - break - if not prefixFound: - return - if '.' not in blogAddress: - return - if ' ' in blogAddress: - return - if ',' in blogAddress: - return - - for propertyValue in actorJson['attachment']: - if not propertyValue.get('name'): - continue - if not propertyValue.get('type'): - continue - if not propertyValue['name'].lower().startswith('blog'): - continue - if propertyValue['type'] != 'PropertyValue': - continue - propertyValue['value'] = blogAddress - return - - newBlogAddress = { - "name": "Blog", - "type": "PropertyValue", - "value": blogAddress - } - actorJson['attachment'].append(newBlogAddress) - - -def updateAvatarImageCache(session, baseDir: str, httpPrefix: str, - actor: str, avatarUrl: str, - personCache: {}, allowDownloads: bool, - force=False) -> str: - """Updates the cached avatar for the given actor - """ - if not avatarUrl: - return None - actorStr = actor.replace('/', '-') - avatarImagePath = baseDir + '/cache/avatars/' + actorStr - if avatarUrl.endswith('.png') or \ - '.png?' in avatarUrl: - sessionHeaders = { - 'Accept': 'image/png' - } - avatarImageFilename = avatarImagePath + '.png' - elif (avatarUrl.endswith('.jpg') or - avatarUrl.endswith('.jpeg') or - '.jpg?' in avatarUrl or - '.jpeg?' in avatarUrl): - sessionHeaders = { - 'Accept': 'image/jpeg' - } - avatarImageFilename = avatarImagePath + '.jpg' - elif avatarUrl.endswith('.gif') or '.gif?' in avatarUrl: - sessionHeaders = { - 'Accept': 'image/gif' - } - avatarImageFilename = avatarImagePath + '.gif' - elif avatarUrl.endswith('.webp') or '.webp?' in avatarUrl: - sessionHeaders = { - 'Accept': 'image/webp' - } - avatarImageFilename = avatarImagePath + '.webp' - elif avatarUrl.endswith('.avif') or '.avif?' in avatarUrl: - sessionHeaders = { - 'Accept': 'image/avif' - } - avatarImageFilename = avatarImagePath + '.avif' - else: - return None - - if (not os.path.isfile(avatarImageFilename) or force) and allowDownloads: - try: - print('avatar image url: ' + avatarUrl) - result = session.get(avatarUrl, - headers=sessionHeaders, - params=None) - if result.status_code < 200 or \ - result.status_code > 202: - print('Avatar image download failed with status ' + - str(result.status_code)) - # remove partial download - if os.path.isfile(avatarImageFilename): - os.remove(avatarImageFilename) - else: - with open(avatarImageFilename, 'wb') as f: - f.write(result.content) - print('avatar image downloaded for ' + actor) - return avatarImageFilename.replace(baseDir + '/cache', '') - except Exception as e: - print('Failed to download avatar image: ' + str(avatarUrl)) - print(e) - prof = 'https://www.w3.org/ns/activitystreams' - if '/channel/' not in actor or '/accounts/' not in actor: - sessionHeaders = { - 'Accept': 'application/activity+json; profile="' + prof + '"' - } - else: - sessionHeaders = { - 'Accept': 'application/ld+json; profile="' + prof + '"' - } - personJson = \ - getJson(session, actor, sessionHeaders, None, __version__, - httpPrefix, None) - if personJson: - if not personJson.get('id'): - return None - if not personJson.get('publicKey'): - return None - if not personJson['publicKey'].get('publicKeyPem'): - return None - if personJson['id'] != actor: - return None - if not personCache.get(actor): - return None - if personCache[actor]['actor']['publicKey']['publicKeyPem'] != \ - personJson['publicKey']['publicKeyPem']: - print("ERROR: " + - "public keys don't match when downloading actor for " + - actor) - return None - storePersonInCache(baseDir, actor, personJson, personCache, - allowDownloads) - return getPersonAvatarUrl(baseDir, actor, personCache, - allowDownloads) - return None - return avatarImageFilename.replace(baseDir + '/cache', '') - - -def getPersonAvatarUrl(baseDir: str, personUrl: str, personCache: {}, - allowDownloads: bool) -> str: - """Returns the avatar url for the person - """ - personJson = \ - getPersonFromCache(baseDir, personUrl, personCache, allowDownloads) - if not personJson: - return None - - # get from locally stored image - actorStr = personJson['id'].replace('/', '-') - avatarImagePath = baseDir + '/cache/avatars/' + actorStr - - imageExtension = ('png', 'jpg', 'jpeg', 'gif', 'webp', 'avif') - for ext in imageExtension: - if os.path.isfile(avatarImagePath + '.' + ext): - return '/avatars/' + actorStr + '.' + ext - elif os.path.isfile(avatarImagePath.lower() + '.' + ext): - return '/avatars/' + actorStr.lower() + '.' + ext - - if personJson.get('icon'): - if personJson['icon'].get('url'): - return personJson['icon']['url'] - return None +from feeds import rss2TagHeader +from feeds import rss2TagFooter +from webapp_utils import getAltPath +from webapp_utils import getContentWarningButton +from webapp_utils import getBlogAddress +from webapp_utils import getPersonAvatarUrl +from webapp_utils import updateAvatarImageCache +from webapp_utils import getIconsDir +from webapp_utils import scheduledPostsExist +from webapp_utils import sharesTimelineJson +from webapp_utils import postContainsPublic +from webapp_utils import isQuestion def htmlFollowingList(cssCache: {}, baseDir: str, @@ -460,17 +230,6 @@ def htmlSearchEmoji(cssCache: {}, translate: {}, return emojiForm -def getIconsDir(baseDir: str) -> str: - """Returns the directory where icons exist - """ - iconsDir = 'icons' - theme = getConfigParam(baseDir, 'theme') - if theme: - if os.path.isdir(baseDir + '/img/icons/' + theme): - iconsDir = 'icons/' + theme - return iconsDir - - def htmlSearchSharedItems(cssCache: {}, translate: {}, baseDir: str, searchStr: str, pageNumber: int, @@ -863,22 +622,6 @@ def htmlHashtagSearch(cssCache: {}, return hashtagSearchForm -def rss2TagHeader(hashtag: str, httpPrefix: str, domainFull: str) -> str: - rssStr = "" - rssStr += "" - rssStr += '' - rssStr += ' #' + hashtag + '' - rssStr += ' ' + httpPrefix + '://' + domainFull + \ - '/tags/rss2/' + hashtag + '' - return rssStr - - -def rss2TagFooter() -> str: - rssStr = '' - rssStr += '' - return rssStr - - def rssHashtagSearch(nickname: str, domain: str, port: int, recentPostsCache: {}, maxRecentPosts: int, translate: {}, @@ -1216,18 +959,6 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, return historySearchForm -def scheduledPostsExist(baseDir: str, nickname: str, domain: str) -> bool: - """Returns true if there are posts scheduled to be delivered - """ - scheduleIndexFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + '/schedule.index' - if not os.path.isfile(scheduleIndexFilename): - return False - if '#users#' in open(scheduleIndexFilename).read(): - return True - return False - - def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, domain: str, port: int, httpPrefix: str, defaultTimeline: str) -> str: @@ -3313,58 +3044,6 @@ def htmlProfileShares(actor: str, translate: {}, return profileStr -def sharesTimelineJson(actor: str, pageNumber: int, itemsPerPage: int, - baseDir: str, maxSharesPerAccount: int) -> ({}, bool): - """Get a page on the shared items timeline as json - maxSharesPerAccount helps to avoid one person dominating the timeline - by sharing a large number of things - """ - allSharesJson = {} - for subdir, dirs, files in os.walk(baseDir + '/accounts'): - for handle in dirs: - if '@' in handle: - accountDir = baseDir + '/accounts/' + handle - sharesFilename = accountDir + '/shares.json' - if os.path.isfile(sharesFilename): - sharesJson = loadJson(sharesFilename) - if not sharesJson: - continue - nickname = handle.split('@')[0] - # actor who owns this share - owner = actor.split('/users/')[0] + '/users/' + nickname - ctr = 0 - for itemID, item in sharesJson.items(): - # assign owner to the item - item['actor'] = owner - allSharesJson[str(item['published'])] = item - ctr += 1 - if ctr >= maxSharesPerAccount: - break - # sort the shared items in descending order of publication date - sharesJson = OrderedDict(sorted(allSharesJson.items(), reverse=True)) - lastPage = False - startIndex = itemsPerPage*pageNumber - maxIndex = len(sharesJson.items()) - if maxIndex < itemsPerPage: - lastPage = True - if startIndex >= maxIndex - itemsPerPage: - lastPage = True - startIndex = maxIndex - itemsPerPage - if startIndex < 0: - startIndex = 0 - ctr = 0 - resultJson = {} - for published, item in sharesJson.items(): - if ctr >= startIndex + itemsPerPage: - break - if ctr < startIndex: - ctr += 1 - continue - resultJson[published] = item - ctr += 1 - return resultJson, lastPage - - def htmlSharesTimeline(translate: {}, pageNumber: int, itemsPerPage: int, baseDir: str, actor: str, nickname: str, domain: str, port: int, @@ -4213,26 +3892,6 @@ def addEmojiToDisplayName(baseDir: str, httpPrefix: str, return displayName -def postContainsPublic(postJsonObject: {}) -> bool: - """Does the given post contain #Public - """ - containsPublic = False - if not postJsonObject['object'].get('to'): - return containsPublic - - for toAddress in postJsonObject['object']['to']: - if toAddress.endswith('#Public'): - containsPublic = True - break - if not containsPublic: - if postJsonObject['object'].get('cc'): - for toAddress in postJsonObject['object']['cc']: - if toAddress.endswith('#Public'): - containsPublic = True - break - return containsPublic - - def loadIndividualPostAsHtmlFromCache(baseDir: str, nickname: str, domain: str, postJsonObject: {}) -> str: @@ -5648,25 +5307,6 @@ def individualPostAsHtml(allowDownloads: bool, return postHtml -def isQuestion(postObjectJson: {}) -> bool: - """ is the given post a question? - """ - if postObjectJson['type'] != 'Create' and \ - postObjectJson['type'] != 'Update': - return False - if not isinstance(postObjectJson['object'], dict): - return False - if not postObjectJson['object'].get('type'): - return False - if postObjectJson['object']['type'] != 'Question': - return False - if not postObjectJson['object'].get('oneOf'): - return False - if not isinstance(postObjectJson['object']['oneOf'], list): - return False - return True - - def htmlHighlightLabel(label: str, highlight: bool) -> str: """If the give text should be highlighted then return the appropriate markup. diff --git a/webapp_utils.py b/webapp_utils.py new file mode 100644 index 000000000..eb35bf144 --- /dev/null +++ b/webapp_utils.py @@ -0,0 +1,391 @@ +__filename__ = "webapp_utils.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.1.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import os +from collections import OrderedDict +from utils import getProtocolPrefixes +from utils import loadJson +from utils import getJson +from utils import getConfigParam +from cache import getPersonFromCache +from cache import storePersonInCache + + +def getAltPath(actor: str, domainFull: str, callingDomain: str) -> str: + """Returns alternate path from the actor + eg. https://clearnetdomain/path becomes http://oniondomain/path + """ + postActor = actor + if callingDomain not in actor and domainFull in actor: + if callingDomain.endswith('.onion') or \ + callingDomain.endswith('.i2p'): + postActor = \ + 'http://' + callingDomain + actor.split(domainFull)[1] + print('Changed POST domain from ' + actor + ' to ' + postActor) + return postActor + + +def getContentWarningButton(postID: str, translate: {}, + content: str) -> str: + """Returns the markup for a content warning button + """ + return '
' + \ + translate['SHOW MORE'] + '' + \ + '
' + content + \ + '
\n' + + +def getActorPropertyUrl(actorJson: {}, propertyName: str) -> str: + """Returns a url property from an actor + """ + if not actorJson.get('attachment'): + return '' + propertyName = propertyName.lower() + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue['name'].lower().startswith(propertyName): + continue + if not propertyValue.get('type'): + continue + if not propertyValue.get('value'): + continue + if propertyValue['type'] != 'PropertyValue': + continue + propertyValue['value'] = propertyValue['value'].strip() + prefixes = getProtocolPrefixes() + prefixFound = False + for prefix in prefixes: + if propertyValue['value'].startswith(prefix): + prefixFound = True + break + if not prefixFound: + continue + if '.' not in propertyValue['value']: + continue + if ' ' in propertyValue['value']: + continue + if ',' in propertyValue['value']: + continue + return propertyValue['value'] + return '' + + +def getBlogAddress(actorJson: {}) -> str: + """Returns blog address for the given actor + """ + return getActorPropertyUrl(actorJson, 'Blog') + + +def setActorPropertyUrl(actorJson: {}, propertyName: str, url: str) -> None: + """Sets a url for the given actor property + """ + if not actorJson.get('attachment'): + actorJson['attachment'] = [] + + propertyNameLower = propertyName.lower() + + # remove any existing value + propertyFound = None + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue.get('type'): + continue + if not propertyValue['name'].lower().startswith(propertyNameLower): + continue + propertyFound = propertyValue + break + if propertyFound: + actorJson['attachment'].remove(propertyFound) + + prefixes = getProtocolPrefixes() + prefixFound = False + for prefix in prefixes: + if url.startswith(prefix): + prefixFound = True + break + if not prefixFound: + return + if '.' not in url: + return + if ' ' in url: + return + if ',' in url: + return + + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue.get('type'): + continue + if not propertyValue['name'].lower().startswith(propertyNameLower): + continue + if propertyValue['type'] != 'PropertyValue': + continue + propertyValue['value'] = url + return + + newAddress = { + "name": propertyName, + "type": "PropertyValue", + "value": url + } + actorJson['attachment'].append(newAddress) + + +def setBlogAddress(actorJson: {}, blogAddress: str) -> None: + """Sets an blog address for the given actor + """ + setActorPropertyUrl(actorJson, 'Blog', blogAddress) + + +def updateAvatarImageCache(session, baseDir: str, httpPrefix: str, + actor: str, avatarUrl: str, + personCache: {}, allowDownloads: bool, + force=False) -> str: + """Updates the cached avatar for the given actor + """ + if not avatarUrl: + return None + actorStr = actor.replace('/', '-') + avatarImagePath = baseDir + '/cache/avatars/' + actorStr + if avatarUrl.endswith('.png') or \ + '.png?' in avatarUrl: + sessionHeaders = { + 'Accept': 'image/png' + } + avatarImageFilename = avatarImagePath + '.png' + elif (avatarUrl.endswith('.jpg') or + avatarUrl.endswith('.jpeg') or + '.jpg?' in avatarUrl or + '.jpeg?' in avatarUrl): + sessionHeaders = { + 'Accept': 'image/jpeg' + } + avatarImageFilename = avatarImagePath + '.jpg' + elif avatarUrl.endswith('.gif') or '.gif?' in avatarUrl: + sessionHeaders = { + 'Accept': 'image/gif' + } + avatarImageFilename = avatarImagePath + '.gif' + elif avatarUrl.endswith('.webp') or '.webp?' in avatarUrl: + sessionHeaders = { + 'Accept': 'image/webp' + } + avatarImageFilename = avatarImagePath + '.webp' + elif avatarUrl.endswith('.avif') or '.avif?' in avatarUrl: + sessionHeaders = { + 'Accept': 'image/avif' + } + avatarImageFilename = avatarImagePath + '.avif' + else: + return None + + if (not os.path.isfile(avatarImageFilename) or force) and allowDownloads: + try: + print('avatar image url: ' + avatarUrl) + result = session.get(avatarUrl, + headers=sessionHeaders, + params=None) + if result.status_code < 200 or \ + result.status_code > 202: + print('Avatar image download failed with status ' + + str(result.status_code)) + # remove partial download + if os.path.isfile(avatarImageFilename): + os.remove(avatarImageFilename) + else: + with open(avatarImageFilename, 'wb') as f: + f.write(result.content) + print('avatar image downloaded for ' + actor) + return avatarImageFilename.replace(baseDir + '/cache', '') + except Exception as e: + print('Failed to download avatar image: ' + str(avatarUrl)) + print(e) + prof = 'https://www.w3.org/ns/activitystreams' + if '/channel/' not in actor or '/accounts/' not in actor: + sessionHeaders = { + 'Accept': 'application/activity+json; profile="' + prof + '"' + } + else: + sessionHeaders = { + 'Accept': 'application/ld+json; profile="' + prof + '"' + } + personJson = \ + getJson(session, actor, sessionHeaders, None, __version__, + httpPrefix, None) + if personJson: + if not personJson.get('id'): + return None + if not personJson.get('publicKey'): + return None + if not personJson['publicKey'].get('publicKeyPem'): + return None + if personJson['id'] != actor: + return None + if not personCache.get(actor): + return None + if personCache[actor]['actor']['publicKey']['publicKeyPem'] != \ + personJson['publicKey']['publicKeyPem']: + print("ERROR: " + + "public keys don't match when downloading actor for " + + actor) + return None + storePersonInCache(baseDir, actor, personJson, personCache, + allowDownloads) + return getPersonAvatarUrl(baseDir, actor, personCache, + allowDownloads) + return None + return avatarImageFilename.replace(baseDir + '/cache', '') + + +def getImageExtensions() -> []: + """Returns a list of the possible image file extensions + """ + return ('png', 'jpg', 'jpeg', 'gif', 'webp', 'avif') + + +def getPersonAvatarUrl(baseDir: str, personUrl: str, personCache: {}, + allowDownloads: bool) -> str: + """Returns the avatar url for the person + """ + personJson = \ + getPersonFromCache(baseDir, personUrl, personCache, allowDownloads) + if not personJson: + return None + + # get from locally stored image + actorStr = personJson['id'].replace('/', '-') + avatarImagePath = baseDir + '/cache/avatars/' + actorStr + + imageExtension = getImageExtensions() + for ext in imageExtension: + if os.path.isfile(avatarImagePath + '.' + ext): + return '/avatars/' + actorStr + '.' + ext + elif os.path.isfile(avatarImagePath.lower() + '.' + ext): + return '/avatars/' + actorStr.lower() + '.' + ext + + if personJson.get('icon'): + if personJson['icon'].get('url'): + return personJson['icon']['url'] + return None + + +def getIconsDir(baseDir: str) -> str: + """Returns the directory where icons exist + """ + iconsDir = 'icons' + theme = getConfigParam(baseDir, 'theme') + if theme: + if os.path.isdir(baseDir + '/img/icons/' + theme): + iconsDir = 'icons/' + theme + return iconsDir + + +def scheduledPostsExist(baseDir: str, nickname: str, domain: str) -> bool: + """Returns true if there are posts scheduled to be delivered + """ + scheduleIndexFilename = \ + baseDir + '/accounts/' + nickname + '@' + domain + '/schedule.index' + if not os.path.isfile(scheduleIndexFilename): + return False + if '#users#' in open(scheduleIndexFilename).read(): + return True + return False + + +def sharesTimelineJson(actor: str, pageNumber: int, itemsPerPage: int, + baseDir: str, maxSharesPerAccount: int) -> ({}, bool): + """Get a page on the shared items timeline as json + maxSharesPerAccount helps to avoid one person dominating the timeline + by sharing a large number of things + """ + allSharesJson = {} + for subdir, dirs, files in os.walk(baseDir + '/accounts'): + for handle in dirs: + if '@' in handle: + accountDir = baseDir + '/accounts/' + handle + sharesFilename = accountDir + '/shares.json' + if os.path.isfile(sharesFilename): + sharesJson = loadJson(sharesFilename) + if not sharesJson: + continue + nickname = handle.split('@')[0] + # actor who owns this share + owner = actor.split('/users/')[0] + '/users/' + nickname + ctr = 0 + for itemID, item in sharesJson.items(): + # assign owner to the item + item['actor'] = owner + allSharesJson[str(item['published'])] = item + ctr += 1 + if ctr >= maxSharesPerAccount: + break + # sort the shared items in descending order of publication date + sharesJson = OrderedDict(sorted(allSharesJson.items(), reverse=True)) + lastPage = False + startIndex = itemsPerPage * pageNumber + maxIndex = len(sharesJson.items()) + if maxIndex < itemsPerPage: + lastPage = True + if startIndex >= maxIndex - itemsPerPage: + lastPage = True + startIndex = maxIndex - itemsPerPage + if startIndex < 0: + startIndex = 0 + ctr = 0 + resultJson = {} + for published, item in sharesJson.items(): + if ctr >= startIndex + itemsPerPage: + break + if ctr < startIndex: + ctr += 1 + continue + resultJson[published] = item + ctr += 1 + return resultJson, lastPage + + +def postContainsPublic(postJsonObject: {}) -> bool: + """Does the given post contain #Public + """ + containsPublic = False + if not postJsonObject['object'].get('to'): + return containsPublic + + for toAddress in postJsonObject['object']['to']: + if toAddress.endswith('#Public'): + containsPublic = True + break + if not containsPublic: + if postJsonObject['object'].get('cc'): + for toAddress in postJsonObject['object']['cc']: + if toAddress.endswith('#Public'): + containsPublic = True + break + return containsPublic + + +def isQuestion(postObjectJson: {}) -> bool: + """ is the given post a question? + """ + if postObjectJson['type'] != 'Create' and \ + postObjectJson['type'] != 'Update': + return False + if not isinstance(postObjectJson['object'], dict): + return False + if not postObjectJson['object'].get('type'): + return False + if postObjectJson['object']['type'] != 'Question': + return False + if not postObjectJson['object'].get('oneOf'): + return False + if not isinstance(postObjectJson['object']['oneOf'], list): + return False + return True From 1401167fe0d10f8e1d4b289e6c2344e096a23151 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 9 Nov 2020 15:25:18 +0000 Subject: [PATCH 19/25] Fix module --- webapp_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_utils.py b/webapp_utils.py index eb35bf144..f18e6e749 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -8,9 +8,9 @@ __status__ = "Production" import os from collections import OrderedDict +from session import getJson from utils import getProtocolPrefixes from utils import loadJson -from utils import getJson from utils import getConfigParam from cache import getPersonFromCache from cache import storePersonInCache From d34d22dc761a75ebf0c149fb8d9457479571e224 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 9 Nov 2020 15:40:24 +0000 Subject: [PATCH 20/25] Moving functions out of webapp --- follow.py | 14 +++++++++++ question.py | 19 ++++++++++++++ webapp.py | 66 ++++++------------------------------------------- webapp_utils.py | 57 ++++++++++++++++++++++++++++++------------ 4 files changed, 81 insertions(+), 75 deletions(-) diff --git a/follow.py b/follow.py index 9f61636eb..9403f92ea 100644 --- a/follow.py +++ b/follow.py @@ -1232,3 +1232,17 @@ def outboxUndoFollow(baseDir: str, messageJson: {}, debug: bool) -> None: if debug: print('WARN: ' + nicknameFollower + ' could not unfollow ' + nicknameFollowing + '@' + domainFollowingFull) + + +def followerApprovalActive(baseDir: str, nickname: str, domain: str) -> bool: + """Returns true if the given account requires follower approval + """ + manuallyApprovesFollowers = False + actorFilename = baseDir + '/accounts/' + nickname + '@' + domain + '.json' + if os.path.isfile(actorFilename): + actorJson = loadJson(actorFilename) + if actorJson: + if actorJson.get('manuallyApprovesFollowers'): + manuallyApprovesFollowers = \ + actorJson['manuallyApprovesFollowers'] + return manuallyApprovesFollowers diff --git a/question.py b/question.py index 9618154ae..31e83f29c 100644 --- a/question.py +++ b/question.py @@ -124,3 +124,22 @@ def questionUpdateVotes(baseDir: str, nickname: str, domain: str, # save the question with altered totals saveJson(questionJson, questionPostFilename) return questionJson + + +def isQuestion(postObjectJson: {}) -> bool: + """ is the given post a question? + """ + if postObjectJson['type'] != 'Create' and \ + postObjectJson['type'] != 'Update': + return False + if not isinstance(postObjectJson['object'], dict): + return False + if not postObjectJson['object'].get('type'): + return False + if postObjectJson['object']['type'] != 'Question': + return False + if not postObjectJson['object'].get('oneOf'): + return False + if not isinstance(postObjectJson['object']['oneOf'], list): + return False + return True diff --git a/webapp.py b/webapp.py index a3a9be795..dbfff8c99 100644 --- a/webapp.py +++ b/webapp.py @@ -24,6 +24,7 @@ from ssb import getSSBAddress from tox import getToxAddress from matrix import getMatrixAddress from donate import getDonationUrl +from question import isQuestion from utils import firstParagraphFromString from utils import getCSS from utils import isSystemAccount @@ -48,6 +49,7 @@ from utils import getConfigParam from utils import votesOnNewswireItem from utils import removeHtml from follow import isFollowingActor +from follow import followerApprovalActive from webfinger import webfingerHandle from posts import isDM from posts import getPersonBox @@ -96,7 +98,11 @@ from webapp_utils import getIconsDir from webapp_utils import scheduledPostsExist from webapp_utils import sharesTimelineJson from webapp_utils import postContainsPublic -from webapp_utils import isQuestion +from webapp_utils import getImageFile +from webapp_utils import getBannerFile +from webapp_utils import getSearchBannerFile +from webapp_utils import getLeftImageFile +from webapp_utils import getRightImageFile def htmlFollowingList(cssCache: {}, baseDir: str, @@ -3741,20 +3747,6 @@ def addEmbeddedElements(translate: {}, content: str) -> str: return addEmbeddedVideo(translate, content) -def followerApprovalActive(baseDir: str, nickname: str, domain: str) -> bool: - """Returns true if the given account requires follower approval - """ - manuallyApprovesFollowers = False - actorFilename = baseDir + '/accounts/' + nickname + '@' + domain + '.json' - if os.path.isfile(actorFilename): - actorJson = loadJson(actorFilename) - if actorJson: - if actorJson.get('manuallyApprovesFollowers'): - manuallyApprovesFollowers = \ - actorJson['manuallyApprovesFollowers'] - return manuallyApprovesFollowers - - def insertQuestion(baseDir: str, translate: {}, nickname: str, domain: str, port: int, content: str, @@ -5950,50 +5942,6 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, return htmlStr -def getImageFile(baseDir: str, name: str, directory: str, - nickname: str, domain: str) -> (str, str): - """ - returns the filenames for an image with the given name - """ - bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') - bannerFile = '' - bannerFilename = '' - for ext in bannerExtensions: - bannerFile = name + '.' + ext - bannerFilename = directory + '/' + bannerFile - if os.path.isfile(bannerFilename): - break - return bannerFile, bannerFilename - - -def getBannerFile(baseDir: str, - nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'banner', - baseDir + '/accounts/' + nickname + '@' + domain, - nickname, domain) - - -def getSearchBannerFile(baseDir: str, - nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'search_banner', - baseDir + '/accounts/' + nickname + '@' + domain, - nickname, domain) - - -def getLeftImageFile(baseDir: str, - nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'left_col_image', - baseDir + '/accounts/' + nickname + '@' + domain, - nickname, domain) - - -def getRightImageFile(baseDir: str, - nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'right_col_image', - baseDir + '/accounts/' + nickname + '@' + domain, - nickname, domain) - - def headerButtonsFrontScreen(translate: {}, nickname: str, boxName: str, authorized: bool, diff --git a/webapp_utils.py b/webapp_utils.py index f18e6e749..e92d3a212 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -372,20 +372,45 @@ def postContainsPublic(postJsonObject: {}) -> bool: return containsPublic -def isQuestion(postObjectJson: {}) -> bool: - """ is the given post a question? +def getImageFile(baseDir: str, name: str, directory: str, + nickname: str, domain: str) -> (str, str): """ - if postObjectJson['type'] != 'Create' and \ - postObjectJson['type'] != 'Update': - return False - if not isinstance(postObjectJson['object'], dict): - return False - if not postObjectJson['object'].get('type'): - return False - if postObjectJson['object']['type'] != 'Question': - return False - if not postObjectJson['object'].get('oneOf'): - return False - if not isinstance(postObjectJson['object']['oneOf'], list): - return False - return True + returns the filenames for an image with the given name + """ + bannerExtensions = getImageExtensions() + bannerFile = '' + bannerFilename = '' + for ext in bannerExtensions: + bannerFile = name + '.' + ext + bannerFilename = directory + '/' + bannerFile + if os.path.isfile(bannerFilename): + break + return bannerFile, bannerFilename + + +def getBannerFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + +def getSearchBannerFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'search_banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + +def getLeftImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'left_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + +def getRightImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'right_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) From a3a022c917b22606457d702de3c9c4703f0f0b07 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 9 Nov 2020 19:41:01 +0000 Subject: [PATCH 21/25] Splitting webapp into smaller modules --- blog.py | 4 +- daemon.py | 16 +- delete.py | 32 + inbox.py | 2 +- posts.py | 24 + utils.py | 8 + webapp.py | 2838 +------------------------------------------- webapp_media.py | 224 ++++ webapp_question.py | 104 ++ webapp_search.py | 967 +++++++++++++++ webapp_utils.py | 326 +++++ 11 files changed, 1703 insertions(+), 2842 deletions(-) create mode 100644 webapp_media.py create mode 100644 webapp_question.py create mode 100644 webapp_search.py diff --git a/blog.py b/blog.py index 2f0d4e70a..9963ae0ea 100644 --- a/blog.py +++ b/blog.py @@ -11,10 +11,10 @@ from datetime import datetime from content import replaceEmojiFromTags from webapp import getIconsDir -from webapp import getPostAttachmentsAsHtml from webapp import htmlHeader from webapp import htmlFooter -from webapp import addEmbeddedElements +from webapp_media import addEmbeddedElements +from webapp_utils import getPostAttachmentsAsHtml from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost diff --git a/daemon.py b/daemon.py index 967aac905..f9c85e10d 100644 --- a/daemon.py +++ b/daemon.py @@ -143,11 +143,8 @@ from webapp import htmlGetLoginCredentials from webapp import htmlNewPost from webapp import htmlFollowConfirm from webapp import htmlCalendar -from webapp import htmlSearch from webapp import htmlNewswireMobile from webapp import htmlLinksMobile -from webapp import htmlSearchEmoji -from webapp import htmlSearchEmojiTextEntry from webapp import htmlUnfollowConfirm from webapp import htmlProfileAfterSearch from webapp import htmlEditProfile @@ -155,13 +152,16 @@ from webapp import htmlEditLinks from webapp import htmlEditNewswire from webapp import htmlEditNewsPost from webapp import htmlTermsOfService -from webapp import htmlSkillsSearch -from webapp import htmlHistorySearch -from webapp import htmlHashtagSearch -from webapp import rssHashtagSearch from webapp import htmlModerationInfo -from webapp import htmlSearchSharedItems from webapp import htmlHashtagBlocked +from webapp_search import htmlSkillsSearch +from webapp_search import htmlHistorySearch +from webapp_search import htmlHashtagSearch +from webapp_search import rssHashtagSearch +from webapp_search import htmlSearchEmoji +from webapp_search import htmlSearchSharedItems +from webapp_search import htmlSearchEmojiTextEntry +from webapp_search import htmlSearch from shares import getSharesFeedForPerson from shares import addShare from shares import removeShare diff --git a/delete.py b/delete.py index 751b95aa0..3b195db7e 100644 --- a/delete.py +++ b/delete.py @@ -6,6 +6,8 @@ __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" +import os +from datetime import datetime from utils import removeIdEnding from utils import getStatusNumber from utils import urlPermitted @@ -295,3 +297,33 @@ def outboxDelete(baseDir: str, httpPrefix: str, postFilename, debug, recentPostsCache) if debug: print('DEBUG: post deleted via c2s - ' + postFilename) + + +def removeOldHashtags(baseDir: str, maxMonths: int) -> str: + """Remove old hashtags + """ + if maxMonths > 11: + maxMonths = 11 + maxDaysSinceEpoch = \ + (datetime.utcnow() - datetime(1970, 1 + maxMonths, 1)).days + removeHashtags = [] + + for subdir, dirs, files in os.walk(baseDir + '/tags'): + for f in files: + tagsFilename = os.path.join(baseDir + '/tags', f) + if not os.path.isfile(tagsFilename): + continue + # get last modified datetime + modTimesinceEpoc = os.path.getmtime(tagsFilename) + lastModifiedDate = datetime.fromtimestamp(modTimesinceEpoc) + fileDaysSinceEpoch = (lastModifiedDate - datetime(1970, 1, 1)).days + + # check of the file is too old + if fileDaysSinceEpoch < maxDaysSinceEpoch: + removeHashtags.append(tagsFilename) + + for removeFilename in removeHashtags: + try: + os.remove(removeFilename) + except BaseException: + pass diff --git a/inbox.py b/inbox.py index 4916af2fa..5deb64b2a 100644 --- a/inbox.py +++ b/inbox.py @@ -58,7 +58,6 @@ from posts import sendSignedJson from posts import sendToFollowersThread from webapp import individualPostAsHtml from webapp import getIconsDir -from webapp import removeOldHashtags from question import questionUpdateVotes from media import replaceYouTube from git import isGitPatch @@ -66,6 +65,7 @@ from git import receiveGitPatch from followingCalendar import receivingCalendarEvents from content import dangerousMarkup from happening import saveEventPost +from delete import removeOldHashtags def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: diff --git a/posts.py b/posts.py index ec59bc6db..5e16b369b 100644 --- a/posts.py +++ b/posts.py @@ -3977,3 +3977,27 @@ def sendUndoBlockViaServer(baseDir: str, session, print('DEBUG: c2s POST block success') return newBlockJson + + +def postIsMuted(baseDir: str, nickname: str, domain: str, + postJsonObject: {}, messageId: str) -> bool: + """ Returns true if the given post is muted + """ + isMuted = postJsonObject.get('muted') + if isMuted is True or isMuted is False: + return isMuted + postDir = baseDir + '/accounts/' + nickname + '@' + domain + muteFilename = \ + postDir + '/inbox/' + messageId.replace('/', '#') + '.json.muted' + if os.path.isfile(muteFilename): + return True + muteFilename = \ + postDir + '/outbox/' + messageId.replace('/', '#') + '.json.muted' + if os.path.isfile(muteFilename): + return True + muteFilename = \ + baseDir + '/accounts/cache/announce/' + nickname + \ + '/' + messageId.replace('/', '#') + '.json.muted' + if os.path.isfile(muteFilename): + return True + return False diff --git a/utils.py b/utils.py index b653d280c..23de5d1cf 100644 --- a/utils.py +++ b/utils.py @@ -1494,3 +1494,11 @@ def siteIsActive(url: str) -> bool: if e.errno == errno.ECONNRESET: print('WARN: connection was reset during siteIsActive') return False + + +def weekDayOfMonthStart(monthNumber: int, year: int) -> int: + """Gets the day number of the first day of the month + 1=sun, 7=sat + """ + firstDayOfMonth = datetime(year, monthNumber, 1, 0, 0) + return int(firstDayOfMonth.strftime("%w")) + 1 diff --git a/webapp.py b/webapp.py index dbfff8c99..b810d8fe9 100644 --- a/webapp.py +++ b/webapp.py @@ -8,10 +8,8 @@ __status__ = "Production" import time import os -import urllib.parse from datetime import datetime from datetime import date -from dateutil.parser import parse from shutil import copyfile from pprint import pprint from person import personBoxJson @@ -24,26 +22,15 @@ from ssb import getSSBAddress from tox import getToxAddress from matrix import getMatrixAddress from donate import getDonationUrl -from question import isQuestion -from utils import firstParagraphFromString +from utils import weekDayOfMonthStart from utils import getCSS from utils import isSystemAccount from utils import removeIdEnding -from utils import getProtocolPrefixes -from utils import searchBoxPosts -from utils import isEventPost -from utils import isBlogPost -from utils import isNewsPost -from utils import updateRecentPostsCache from utils import getNicknameFromActor from utils import getDomainFromActor from utils import locatePost from utils import noOfAccounts -from utils import isPublicPost from utils import isPublicPostFromUrl -from utils import getDisplayName -from utils import getCachedPostDirectory -from utils import getCachedPostFilename from utils import loadJson from utils import getConfigParam from utils import votesOnNewswireItem @@ -51,58 +38,40 @@ from utils import removeHtml from follow import isFollowingActor from follow import followerApprovalActive from webfinger import webfingerHandle -from posts import isDM from posts import getPersonBox from posts import getUserUrl from posts import parseUserFeed from posts import populateRepliesJson from posts import isModerator from posts import isEditor -from posts import downloadAnnounce from session import getJson -from auth import createPassword -from like import likedByPerson -from like import noOfLikes -from bookmarks import bookmarkedByPerson -from announce import announcedByPerson from blocking import isBlocked -from blocking import isBlockedHashtag -from content import htmlReplaceEmailQuote -from content import htmlReplaceQuoteMarks -from content import removeTextFormatting -from content import switchWords -from content import getMentionsFromHtml -from content import addHtmlTags -from content import replaceEmojiFromTags from content import removeLongWords from skills import getSkills -from cache import getPersonFromCache from shares import getValidSharedItemID from happening import todaysEventsCheck from happening import thisWeeksEventsCheck from happening import getCalendarEvents from happening import getTodaysEvents -from git import isGitPatch from theme import getThemesList from petnames import getPetName from followingCalendar import receivingCalendarEvents -from devices import E2EEdecryptMessageFromDevice -from feeds import rss2TagHeader -from feeds import rss2TagFooter from webapp_utils import getAltPath -from webapp_utils import getContentWarningButton from webapp_utils import getBlogAddress from webapp_utils import getPersonAvatarUrl -from webapp_utils import updateAvatarImageCache from webapp_utils import getIconsDir from webapp_utils import scheduledPostsExist from webapp_utils import sharesTimelineJson -from webapp_utils import postContainsPublic from webapp_utils import getImageFile from webapp_utils import getBannerFile -from webapp_utils import getSearchBannerFile from webapp_utils import getLeftImageFile from webapp_utils import getRightImageFile +from webapp_utils import htmlHeader +from webapp_utils import htmlFooter +from webapp_utils import addEmojiToDisplayName +from webapp_utils import htmlPostSeparator +from webapp_post import individualPostAsHtml +from webapp_post import preparePostFromHtmlCache def htmlFollowingList(cssCache: {}, baseDir: str, @@ -171,246 +140,6 @@ def htmlFollowingDataList(baseDir: str, nickname: str, return listStr -def htmlSearchEmoji(cssCache: {}, translate: {}, - baseDir: str, httpPrefix: str, - searchStr: str) -> str: - """Search results for emoji - """ - # emoji.json is generated so that it can be customized and the changes - # will be retained even if default_emoji.json is subsequently updated - if not os.path.isfile(baseDir + '/emoji/emoji.json'): - copyfile(baseDir + '/emoji/default_emoji.json', - baseDir + '/emoji/emoji.json') - - searchStr = searchStr.lower().replace(':', '').strip('\n').strip('\r') - cssFilename = baseDir + '/epicyon-profile.css' - if os.path.isfile(baseDir + '/epicyon.css'): - cssFilename = baseDir + '/epicyon.css' - - emojiCSS = getCSS(baseDir, cssFilename, cssCache) - if emojiCSS: - if httpPrefix != 'https': - emojiCSS = emojiCSS.replace('https://', - httpPrefix + '://') - emojiLookupFilename = baseDir + '/emoji/emoji.json' - - # create header - emojiForm = htmlHeader(cssFilename, emojiCSS) - emojiForm += '

' + \ - translate['Emoji Search'] + \ - '

' - - # does the lookup file exist? - if not os.path.isfile(emojiLookupFilename): - emojiForm += '
' + \ - translate['No results'] + '
' - emojiForm += htmlFooter() - return emojiForm - - emojiJson = loadJson(emojiLookupFilename) - if emojiJson: - results = {} - for emojiName, filename in emojiJson.items(): - if searchStr in emojiName: - results[emojiName] = filename + '.png' - for emojiName, filename in emojiJson.items(): - if emojiName in searchStr: - results[emojiName] = filename + '.png' - headingShown = False - emojiForm += '
' - msgStr1 = translate['Copy the text then paste it into your post'] - msgStr2 = ':

' - emojiForm += '' - - emojiForm += htmlFooter() - return emojiForm - - -def htmlSearchSharedItems(cssCache: {}, translate: {}, - baseDir: str, searchStr: str, - pageNumber: int, - resultsPerPage: int, - httpPrefix: str, - domainFull: str, actor: str, - callingDomain: str) -> str: - """Search results for shared items - """ - iconsDir = getIconsDir(baseDir) - currPage = 1 - ctr = 0 - sharedItemsForm = '' - searchStrLower = urllib.parse.unquote(searchStr) - searchStrLower = searchStrLower.lower().strip('\n').strip('\r') - searchStrLowerList = searchStrLower.split('+') - cssFilename = baseDir + '/epicyon-profile.css' - if os.path.isfile(baseDir + '/epicyon.css'): - cssFilename = baseDir + '/epicyon.css' - - sharedItemsCSS = getCSS(baseDir, cssFilename, cssCache) - if sharedItemsCSS: - if httpPrefix != 'https': - sharedItemsCSS = \ - sharedItemsCSS.replace('https://', - httpPrefix + '://') - sharedItemsForm = htmlHeader(cssFilename, sharedItemsCSS) - sharedItemsForm += \ - '

' + translate['Shared Items Search'] + \ - '

' - resultsExist = False - for subdir, dirs, files in os.walk(baseDir + '/accounts'): - for handle in dirs: - if '@' not in handle: - continue - contactNickname = handle.split('@')[0] - sharesFilename = baseDir + '/accounts/' + handle + \ - '/shares.json' - if not os.path.isfile(sharesFilename): - continue - - sharesJson = loadJson(sharesFilename) - if not sharesJson: - continue - - for name, sharedItem in sharesJson.items(): - matched = True - for searchSubstr in searchStrLowerList: - subStrMatched = False - searchSubstr = searchSubstr.strip() - if searchSubstr in sharedItem['location'].lower(): - subStrMatched = True - elif searchSubstr in sharedItem['summary'].lower(): - subStrMatched = True - elif searchSubstr in sharedItem['displayName'].lower(): - subStrMatched = True - elif searchSubstr in sharedItem['category'].lower(): - subStrMatched = True - if not subStrMatched: - matched = False - break - if matched: - if currPage == pageNumber: - sharedItemsForm += '
\n' - sharedItemsForm += \ - '\n' - if sharedItem.get('imageUrl'): - sharedItemsForm += \ - '\n' - sharedItemsForm += \ - 'Item image\n' - sharedItemsForm += \ - '

' + sharedItem['summary'] + '

\n' - sharedItemsForm += \ - '

' + translate['Type'] + \ - ': ' + sharedItem['itemType'] + ' ' - sharedItemsForm += \ - '' + translate['Category'] + \ - ': ' + sharedItem['category'] + ' ' - sharedItemsForm += \ - '' + translate['Location'] + \ - ': ' + sharedItem['location'] + '

\n' - contactActor = \ - httpPrefix + '://' + domainFull + \ - '/users/' + contactNickname - sharedItemsForm += \ - '

\n' - if actor.endswith('/users/' + contactNickname): - sharedItemsForm += \ - ' \n' - sharedItemsForm += '

\n' - if not resultsExist and currPage > 1: - postActor = \ - getAltPath(actor, domainFull, - callingDomain) - # previous page link, needs to be a POST - sharedItemsForm += \ - '
\n' - sharedItemsForm += \ - ' \n' - sharedItemsForm += \ - '
\n' - sharedItemsForm += \ - '
\n' + \ - ' \n' - sharedItemsForm += \ - ' ' + translate['Page up'] + \ - '\n' - sharedItemsForm += '
\n' - sharedItemsForm += '
\n' - resultsExist = True - ctr += 1 - if ctr >= resultsPerPage: - currPage += 1 - if currPage > pageNumber: - postActor = \ - getAltPath(actor, domainFull, - callingDomain) - # next page link, needs to be a POST - sharedItemsForm += \ - '
\n' - sharedItemsForm += \ - ' \n' - sharedItemsForm += \ - '
\n' - sharedItemsForm += \ - '
\n' + \ - ' \n' - sharedItemsForm += \ - ' ' + translate['Page down'] + \ - '\n' - sharedItemsForm += '
\n' - sharedItemsForm += '
\n' - break - ctr = 0 - if not resultsExist: - sharedItemsForm += \ - '
' + translate['No results'] + '
\n' - sharedItemsForm += htmlFooter() - return sharedItemsForm - - def htmlModerationInfo(cssCache: {}, translate: {}, baseDir: str, httpPrefix: str) -> str: msgStr1 = \ @@ -477,494 +206,6 @@ def htmlModerationInfo(cssCache: {}, translate: {}, return infoForm -def htmlHashtagSearch(cssCache: {}, - nickname: str, domain: str, port: int, - recentPostsCache: {}, maxRecentPosts: int, - translate: {}, - baseDir: str, hashtag: str, pageNumber: int, - postsPerPage: int, - session, wfRequest: {}, personCache: {}, - httpPrefix: str, projectVersion: str, - YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: - """Show a page containing search results for a hashtag - """ - if hashtag.startswith('#'): - hashtag = hashtag[1:] - hashtag = urllib.parse.unquote(hashtag) - hashtagIndexFile = baseDir + '/tags/' + hashtag + '.txt' - if not os.path.isfile(hashtagIndexFile): - if hashtag != hashtag.lower(): - hashtag = hashtag.lower() - hashtagIndexFile = baseDir + '/tags/' + hashtag + '.txt' - if not os.path.isfile(hashtagIndexFile): - print('WARN: hashtag file not found ' + hashtagIndexFile) - return None - - iconsDir = getIconsDir(baseDir) - separatorStr = htmlPostSeparator(baseDir, None) - - # check that the directory for the nickname exists - if nickname: - if not os.path.isdir(baseDir + '/accounts/' + - nickname + '@' + domain): - nickname = None - - # read the index - with open(hashtagIndexFile, "r") as f: - lines = f.readlines() - - # read the css - cssFilename = baseDir + '/epicyon-profile.css' - if os.path.isfile(baseDir + '/epicyon.css'): - cssFilename = baseDir + '/epicyon.css' - - hashtagSearchCSS = getCSS(baseDir, cssFilename, cssCache) - if hashtagSearchCSS: - if httpPrefix != 'https': - hashtagSearchCSS = \ - hashtagSearchCSS.replace('https://', - httpPrefix + '://') - - # ensure that the page number is in bounds - if not pageNumber: - pageNumber = 1 - elif pageNumber < 1: - pageNumber = 1 - - # get the start end end within the index file - startIndex = int((pageNumber - 1) * postsPerPage) - endIndex = startIndex + postsPerPage - noOfLines = len(lines) - if endIndex >= noOfLines and noOfLines > 0: - endIndex = noOfLines - 1 - - # add the page title - hashtagSearchForm = htmlHeader(cssFilename, hashtagSearchCSS) - if nickname: - hashtagSearchForm += '
\n' + \ - '

#' + \ - hashtag + '

\n' + '
\n' - else: - hashtagSearchForm += '
\n' + \ - '

#' + hashtag + '

\n' + '
\n' - - # RSS link for hashtag feed - hashtagSearchForm += '
' - hashtagSearchForm += \ - 'RSS 2.0
' - - if startIndex > 0: - # previous page link - hashtagSearchForm += \ - '
\n' + \ - ' ' + translate['Page up'] + \ - '\n
\n' - index = startIndex - while index <= endIndex: - postId = lines[index].strip('\n').strip('\r') - if ' ' not in postId: - nickname = getNicknameFromActor(postId) - if not nickname: - index += 1 - continue - else: - postFields = postId.split(' ') - if len(postFields) != 3: - index += 1 - continue - nickname = postFields[1] - postId = postFields[2] - postFilename = locatePost(baseDir, nickname, domain, postId) - if not postFilename: - index += 1 - continue - postJsonObject = loadJson(postFilename) - if postJsonObject: - if not isPublicPost(postJsonObject): - index += 1 - continue - showIndividualPostIcons = False - if nickname: - showIndividualPostIcons = True - allowDeletion = False - hashtagSearchForm += separatorStr + \ - individualPostAsHtml(True, recentPostsCache, - maxRecentPosts, - iconsDir, translate, None, - baseDir, session, wfRequest, - personCache, - nickname, domain, port, - postJsonObject, - None, True, allowDeletion, - httpPrefix, projectVersion, - 'search', - YTReplacementDomain, - showPublishedDateOnly, - showIndividualPostIcons, - showIndividualPostIcons, - False, False, False) - index += 1 - - if endIndex < noOfLines - 1: - # next page link - hashtagSearchForm += \ - '
\n' + \ - ' ' + translate['Page down'] + '' + \ - '
' - hashtagSearchForm += htmlFooter() - return hashtagSearchForm - - -def rssHashtagSearch(nickname: str, domain: str, port: int, - recentPostsCache: {}, maxRecentPosts: int, - translate: {}, - baseDir: str, hashtag: str, - postsPerPage: int, - session, wfRequest: {}, personCache: {}, - httpPrefix: str, projectVersion: str, - YTReplacementDomain: str) -> str: - """Show an rss feed for a hashtag - """ - if hashtag.startswith('#'): - hashtag = hashtag[1:] - hashtag = urllib.parse.unquote(hashtag) - hashtagIndexFile = baseDir + '/tags/' + hashtag + '.txt' - if not os.path.isfile(hashtagIndexFile): - if hashtag != hashtag.lower(): - hashtag = hashtag.lower() - hashtagIndexFile = baseDir + '/tags/' + hashtag + '.txt' - if not os.path.isfile(hashtagIndexFile): - print('WARN: hashtag file not found ' + hashtagIndexFile) - return None - - # check that the directory for the nickname exists - if nickname: - if not os.path.isdir(baseDir + '/accounts/' + - nickname + '@' + domain): - nickname = None - - # read the index - lines = [] - with open(hashtagIndexFile, "r") as f: - lines = f.readlines() - if not lines: - return None - - domainFull = domain - if port: - if port != 80 and port != 443: - domainFull = domain + ':' + str(port) - - maxFeedLength = 10 - hashtagFeed = \ - rss2TagHeader(hashtag, httpPrefix, domainFull) - for index in range(len(lines)): - postId = lines[index].strip('\n').strip('\r') - if ' ' not in postId: - nickname = getNicknameFromActor(postId) - if not nickname: - index += 1 - if index >= maxFeedLength: - break - continue - else: - postFields = postId.split(' ') - if len(postFields) != 3: - index += 1 - if index >= maxFeedLength: - break - continue - nickname = postFields[1] - postId = postFields[2] - postFilename = locatePost(baseDir, nickname, domain, postId) - if not postFilename: - index += 1 - if index >= maxFeedLength: - break - continue - postJsonObject = loadJson(postFilename) - if postJsonObject: - if not isPublicPost(postJsonObject): - index += 1 - if index >= maxFeedLength: - break - continue - # add to feed - if postJsonObject['object'].get('content') and \ - postJsonObject['object'].get('attributedTo') and \ - postJsonObject['object'].get('published'): - published = postJsonObject['object']['published'] - pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") - rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") - hashtagFeed += ' ' - hashtagFeed += \ - ' ' + \ - postJsonObject['object']['attributedTo'] + \ - '' - if postJsonObject['object'].get('summary'): - hashtagFeed += \ - ' ' + \ - postJsonObject['object']['summary'] + \ - '' - description = postJsonObject['object']['content'] - description = firstParagraphFromString(description) - hashtagFeed += \ - ' ' + description + '' - hashtagFeed += \ - ' ' + rssDateStr + '' - if postJsonObject['object'].get('attachment'): - for attach in postJsonObject['object']['attachment']: - if not attach.get('url'): - continue - hashtagFeed += \ - ' ' + attach['url'] + '' - hashtagFeed += ' ' - index += 1 - if index >= maxFeedLength: - break - - return hashtagFeed + rss2TagFooter() - - -def htmlSkillsSearch(cssCache: {}, translate: {}, baseDir: str, - httpPrefix: str, - skillsearch: str, instanceOnly: bool, - postsPerPage: int) -> str: - """Show a page containing search results for a skill - """ - if skillsearch.startswith('*'): - skillsearch = skillsearch[1:].strip() - - skillsearch = skillsearch.lower().strip('\n').strip('\r') - - results = [] - # search instance accounts - for subdir, dirs, files in os.walk(baseDir + '/accounts/'): - for f in files: - if not f.endswith('.json'): - continue - if '@' not in f: - continue - if f.startswith('inbox@'): - continue - actorFilename = os.path.join(subdir, f) - actorJson = loadJson(actorFilename) - if actorJson: - if actorJson.get('id') and \ - actorJson.get('skills') and \ - actorJson.get('name') and \ - actorJson.get('icon'): - actor = actorJson['id'] - for skillName, skillLevel in actorJson['skills'].items(): - skillName = skillName.lower() - if not (skillName in skillsearch or - skillsearch in skillName): - continue - skillLevelStr = str(skillLevel) - if skillLevel < 100: - skillLevelStr = '0' + skillLevelStr - if skillLevel < 10: - skillLevelStr = '0' + skillLevelStr - indexStr = \ - skillLevelStr + ';' + actor + ';' + \ - actorJson['name'] + \ - ';' + actorJson['icon']['url'] - if indexStr not in results: - results.append(indexStr) - if not instanceOnly: - # search actor cache - for subdir, dirs, files in os.walk(baseDir + '/cache/actors/'): - for f in files: - if not f.endswith('.json'): - continue - if '@' not in f: - continue - if f.startswith('inbox@'): - continue - actorFilename = os.path.join(subdir, f) - cachedActorJson = loadJson(actorFilename) - if cachedActorJson: - if cachedActorJson.get('actor'): - actorJson = cachedActorJson['actor'] - if actorJson.get('id') and \ - actorJson.get('skills') and \ - actorJson.get('name') and \ - actorJson.get('icon'): - actor = actorJson['id'] - for skillName, skillLevel in \ - actorJson['skills'].items(): - skillName = skillName.lower() - if not (skillName in skillsearch or - skillsearch in skillName): - continue - skillLevelStr = str(skillLevel) - if skillLevel < 100: - skillLevelStr = '0' + skillLevelStr - if skillLevel < 10: - skillLevelStr = '0' + skillLevelStr - indexStr = \ - skillLevelStr + ';' + actor + ';' + \ - actorJson['name'] + \ - ';' + actorJson['icon']['url'] - if indexStr not in results: - results.append(indexStr) - - results.sort(reverse=True) - - cssFilename = baseDir + '/epicyon-profile.css' - if os.path.isfile(baseDir + '/epicyon.css'): - cssFilename = baseDir + '/epicyon.css' - - skillSearchCSS = getCSS(baseDir, cssFilename, cssCache) - if skillSearchCSS: - if httpPrefix != 'https': - skillSearchCSS = \ - skillSearchCSS.replace('https://', - httpPrefix + '://') - skillSearchForm = htmlHeader(cssFilename, skillSearchCSS) - skillSearchForm += \ - '

' + translate['Skills search'] + ': ' + \ - skillsearch + '

' - - if len(results) == 0: - skillSearchForm += \ - '
' + translate['No results'] + \ - '
' - else: - skillSearchForm += '
' - ctr = 0 - for skillMatch in results: - skillMatchFields = skillMatch.split(';') - if len(skillMatchFields) != 4: - continue - actor = skillMatchFields[1] - actorName = skillMatchFields[2] - avatarUrl = skillMatchFields[3] - skillSearchForm += \ - '
' - skillSearchForm += \ - '' + actorName + \ - '
' - ctr += 1 - if ctr >= postsPerPage: - break - skillSearchForm += '
' - skillSearchForm += htmlFooter() - return skillSearchForm - - -def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, - httpPrefix: str, - nickname: str, domain: str, - historysearch: str, - postsPerPage: int, pageNumber: int, - projectVersion: str, - recentPostsCache: {}, - maxRecentPosts: int, - session, - wfRequest, - personCache: {}, - port: int, - YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: - """Show a page containing search results for your post history - """ - if historysearch.startswith('!'): - historysearch = historysearch[1:].strip() - - historysearch = historysearch.lower().strip('\n').strip('\r') - - boxFilenames = \ - searchBoxPosts(baseDir, nickname, domain, - historysearch, postsPerPage) - - cssFilename = baseDir + '/epicyon-profile.css' - if os.path.isfile(baseDir + '/epicyon.css'): - cssFilename = baseDir + '/epicyon.css' - - historySearchCSS = getCSS(baseDir, cssFilename, cssCache) - if historySearchCSS: - if httpPrefix != 'https': - historySearchCSS = \ - historySearchCSS.replace('https://', - httpPrefix + '://') - historySearchForm = htmlHeader(cssFilename, historySearchCSS) - - # add the page title - historySearchForm += \ - '

' + translate['Your Posts'] + '

' - - if len(boxFilenames) == 0: - historySearchForm += \ - '
' + translate['No results'] + \ - '
' - return historySearchForm - - iconsDir = getIconsDir(baseDir) - separatorStr = htmlPostSeparator(baseDir, None) - - # ensure that the page number is in bounds - if not pageNumber: - pageNumber = 1 - elif pageNumber < 1: - pageNumber = 1 - - # get the start end end within the index file - startIndex = int((pageNumber - 1) * postsPerPage) - endIndex = startIndex + postsPerPage - noOfBoxFilenames = len(boxFilenames) - if endIndex >= noOfBoxFilenames and noOfBoxFilenames > 0: - endIndex = noOfBoxFilenames - 1 - - index = startIndex - while index <= endIndex: - postFilename = boxFilenames[index] - if not postFilename: - index += 1 - continue - postJsonObject = loadJson(postFilename) - if not postJsonObject: - index += 1 - continue - showIndividualPostIcons = True - allowDeletion = False - historySearchForm += separatorStr + \ - individualPostAsHtml(True, recentPostsCache, - maxRecentPosts, - iconsDir, translate, None, - baseDir, session, wfRequest, - personCache, - nickname, domain, port, - postJsonObject, - None, True, allowDeletion, - httpPrefix, projectVersion, - 'search', - YTReplacementDomain, - showPublishedDateOnly, - showIndividualPostIcons, - showIndividualPostIcons, - False, False, False) - index += 1 - - historySearchForm += htmlFooter() - return historySearchForm - - def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, domain: str, port: int, httpPrefix: str, defaultTimeline: str) -> str: @@ -2831,40 +2072,6 @@ def htmlNewPost(cssCache: {}, mediaInstance: bool, translate: {}, return newPostForm -def getFontFromCss(css: str) -> (str, str): - """Returns the font name and format - """ - if ' url(' not in css: - return None, None - fontName = css.split(" url(")[1].split(")")[0].replace("'", '') - fontFormat = css.split(" format('")[1].split("')")[0] - return fontName, fontFormat - - -def htmlHeader(cssFilename: str, css: str, lang='en') -> str: - htmlStr = '\n' - htmlStr += '\n' - htmlStr += ' \n' - htmlStr += ' \n' - fontName, fontFormat = getFontFromCss(css) - if fontName: - htmlStr += ' \n' - htmlStr += ' \n' - htmlStr += ' \n' - htmlStr += ' \n' - htmlStr += ' Epicyon\n' - htmlStr += ' \n' - htmlStr += ' \n' - return htmlStr - - -def htmlFooter() -> str: - htmlStr = ' \n' - htmlStr += '\n' - return htmlStr - - def htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, translate: {}, baseDir: str, httpPrefix: str, @@ -3530,1775 +2737,6 @@ def individualFollowAsHtml(translate: {}, return resultStr -def addEmbeddedAudio(translate: {}, content: str) -> str: - """Adds embedded audio for mp3/ogg - """ - if not ('.mp3' in content or '.ogg' in content): - return content - - if '