From 96f993d026b96651f2c45c8366f4ab771a37cf28 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 10:11:45 +0000 Subject: [PATCH 01/14] Improve handling of banner images --- webinterface.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/webinterface.py b/webinterface.py index 84b1550b6..7b16d089f 100644 --- a/webinterface.py +++ b/webinterface.py @@ -3557,9 +3557,10 @@ def htmlProfile(rssIconAtTop: bool, # If this is the news account then show a different banner if isSystemAccount(nickname): + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) profileHeaderStr = \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' if loginButton: profileHeaderStr += '
' + loginButton + '
\n' @@ -6000,11 +6001,16 @@ def htmlLinksMobile(cssCache: {}, baseDir: str, else: editor = isEditor(baseDir, nickname) + domain = domainFull + if ':' in domain: + domain = domain.split(':')[0] + htmlStr = htmlHeader(cssFilename, profileStyle) + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) htmlStr += \ '' + \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' htmlStr += '
' + \ headerButtonsFrontScreen(translate, nickname, @@ -6063,10 +6069,12 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, showPublishButton = editor htmlStr = htmlHeader(cssFilename, profileStyle) + + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) htmlStr += \ '' + \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' htmlStr += '
' + \ headerButtonsFrontScreen(translate, nickname, From 32cfdf6f388ff35a7f0d0f6931b3607f58b065b8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 10:42:07 +0000 Subject: [PATCH 02/14] Simplify --- webinterface.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/webinterface.py b/webinterface.py index 7b16d089f..c126f1479 100644 --- a/webinterface.py +++ b/webinterface.py @@ -6092,15 +6092,16 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, return htmlStr -def getBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): +def getImageFile(baseDir: str, name: str, + nickname: str, domain: str) -> (str, str): """ - returns the banner filename + returns the filenames for an image with the given name """ bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') bannerFile = '' bannerFilename = '' for ext in bannerExtensions: - bannerFile = 'banner.' + ext + bannerFile = name + '.' + ext bannerFilename = baseDir + '/accounts/' + \ nickname + '@' + domain + '/' + bannerFile if os.path.isfile(bannerFilename): @@ -6108,21 +6109,14 @@ def getBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): return bannerFile, bannerFilename +def getBannerFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'banner', nickname, domain) + + def getSearchBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): - """ - returns the search banner filename - """ - bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') - bannerFile = '' - bannerFilename = '' - for ext in bannerExtensions: - bannerFile = 'search_banner.' + ext - bannerFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/' + bannerFile - if os.path.isfile(bannerFilename): - break - return bannerFile, bannerFilename + return getImageFile(baseDir, 'search_banner', nickname, domain) def headerButtonsFrontScreen(translate: {}, From 0e1ecd33dc55ea0233ec13382f3fc45415cd8936 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 10:50:38 +0000 Subject: [PATCH 03/14] Function for getting the left column image --- webinterface.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/webinterface.py b/webinterface.py index c126f1479..55fb4ffb9 100644 --- a/webinterface.py +++ b/webinterface.py @@ -5605,9 +5605,8 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, editImageClass = '' if showHeaderImage: - leftColumnImageFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + \ - '/left_col_image.png' + leftImageFile, leftColumnImageFilename = \ + getLeftImageFile(baseDir, nickname, domain) if not os.path.isfile(leftColumnImageFilename): theme = getConfigParam(baseDir, 'theme').lower() if theme == 'default': @@ -5619,6 +5618,7 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, if os.path.isfile(themeLeftColumnImageFilename): copyfile(themeLeftColumnImageFilename, leftColumnImageFilename) + leftImageFile = 'left_col_image.png' # show the image at the top of the column editImageClass = 'leftColEdit' @@ -5628,7 +5628,7 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, '\n
\n' + \ ' \n' + \ + nickname + '/' + leftImageFile + '" />\n' + \ '
\n' if showBackButton: @@ -6119,6 +6119,11 @@ def getSearchBannerFile(baseDir: str, return getImageFile(baseDir, 'search_banner', nickname, domain) +def getLeftImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'left_col_image', nickname, domain) + + def headerButtonsFrontScreen(translate: {}, nickname: str, boxName: str, authorized: bool, From d26d37b51d8da77fcafb0e49f11f5e2302ca038f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 10:53:32 +0000 Subject: [PATCH 04/14] Function for getting the right column image --- webinterface.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/webinterface.py b/webinterface.py index 55fb4ffb9..d7ce87aff 100644 --- a/webinterface.py +++ b/webinterface.py @@ -5859,9 +5859,8 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, # show a column header image, eg. title of the theme or newswire banner editImageClass = '' if showHeaderImage: - rightColumnImageFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + \ - '/right_col_image.png' + rightImageFile, rightColumnImageFilename = \ + getRightImageFile(baseDir, nickname, domain) if not os.path.isfile(rightColumnImageFilename): theme = getConfigParam(baseDir, 'theme').lower() if theme == 'default': @@ -5873,6 +5872,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, if os.path.isfile(themeRightColumnImageFilename): copyfile(themeRightColumnImageFilename, rightColumnImageFilename) + rightImageFile = 'right_col_image.png' # show the image at the top of the column editImageClass = 'rightColEdit' @@ -5882,7 +5882,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, '\n
\n' + \ ' \n' + \ + nickname + '/' + rightImageFile + '" />\n' + \ '
\n' if (showPublishButton or editor or rssIconAtTop) and not showHeaderImage: @@ -6124,6 +6124,11 @@ def getLeftImageFile(baseDir: str, return getImageFile(baseDir, 'left_col_image', nickname, domain) +def getRightImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'right_col_image', nickname, domain) + + def headerButtonsFrontScreen(translate: {}, nickname: str, boxName: str, authorized: bool, From 7369f62a8e0fd1af95abf3c03bd9b624dedb47fb Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 11:06:43 +0000 Subject: [PATCH 05/14] More generic --- webinterface.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/webinterface.py b/webinterface.py index d7ce87aff..d5f0184ce 100644 --- a/webinterface.py +++ b/webinterface.py @@ -5613,12 +5613,16 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, theme = '' else: theme = '_' + theme - themeLeftColumnImageFilename = \ - baseDir + '/img/left_col_image' + theme + '.png' + themeLeftImageFile, themeLeftColumnImageFilename = \ + getImageFile(baseDir, 'left_col_image', baseDir + '/img', + nickname, domain) if os.path.isfile(themeLeftColumnImageFilename): + leftColumnImageFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/' + themeLeftImageFile copyfile(themeLeftColumnImageFilename, leftColumnImageFilename) - leftImageFile = 'left_col_image.png' + leftImageFile = themeLeftImageFile # show the image at the top of the column editImageClass = 'leftColEdit' @@ -6092,7 +6096,7 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, return htmlStr -def getImageFile(baseDir: str, name: str, +def getImageFile(baseDir: str, name: str, directory: str, nickname: str, domain: str) -> (str, str): """ returns the filenames for an image with the given name @@ -6102,8 +6106,7 @@ def getImageFile(baseDir: str, name: str, bannerFilename = '' for ext in bannerExtensions: bannerFile = name + '.' + ext - bannerFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/' + bannerFile + bannerFilename = directory + '/' + bannerFile if os.path.isfile(bannerFilename): break return bannerFile, bannerFilename @@ -6111,22 +6114,30 @@ def getImageFile(baseDir: str, name: str, def getBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'banner', nickname, domain) + return getImageFile(baseDir, 'banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) def getSearchBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'search_banner', nickname, domain) + return getImageFile(baseDir, 'search_banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) def getLeftImageFile(baseDir: str, nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'left_col_image', nickname, domain) + return getImageFile(baseDir, 'left_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) def getRightImageFile(baseDir: str, nickname: str, domain: str) -> (str, str): - return getImageFile(baseDir, 'right_col_image', nickname, domain) + return getImageFile(baseDir, 'right_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) def headerButtonsFrontScreen(translate: {}, From 120a3329c28255b43319fb61c36a59db994303d3 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 11:09:20 +0000 Subject: [PATCH 06/14] More generic --- webinterface.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/webinterface.py b/webinterface.py index d5f0184ce..f4a94a72c 100644 --- a/webinterface.py +++ b/webinterface.py @@ -5871,12 +5871,16 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, theme = '' else: theme = '_' + theme - themeRightColumnImageFilename = \ - baseDir + '/img/right_col_image' + theme + '.png' + themeRightImageFile, themeRightColumnImageFilename = \ + getImageFile(baseDir, 'right_col_image', baseDir + '/img', + nickname, domain) if os.path.isfile(themeRightColumnImageFilename): + rightColumnImageFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/' + themeRightImageFile copyfile(themeRightColumnImageFilename, rightColumnImageFilename) - rightImageFile = 'right_col_image.png' + rightImageFile = themeRightImageFile # show the image at the top of the column editImageClass = 'rightColEdit' From 2721743adec561a1662c405028830e53d78a30be Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 11:23:09 +0000 Subject: [PATCH 07/14] More generic --- webinterface.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/webinterface.py b/webinterface.py index f4a94a72c..9509450b0 100644 --- a/webinterface.py +++ b/webinterface.py @@ -8725,17 +8725,17 @@ def htmlSearch(cssCache: {}, translate: {}, theme = '' else: theme = '_' + theme - bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') - for ext in bannerExtensions: - searchBannerFile = 'search_banner.' + ext + themeSearchImageFile, themeSearchBannerFilename = \ + getImageFile(baseDir, 'search_banner', baseDir + '/img', + searchNickname, domain) + if os.path.isfile(themeSearchBannerFilename): searchBannerFilename = \ baseDir + '/accounts/' + \ - searchNickname + '@' + domain + '/' + searchBannerFile - themeSearchBannerFilename = \ - baseDir + '/img/search_banner' + theme + '.' + ext - if os.path.isfile(themeSearchBannerFilename): - copyfile(themeSearchBannerFilename, searchBannerFilename) - break + searchNickname + '@' + domain + '/' + themeSearchImageFile + copyfile(themeSearchBannerFilename, + searchBannerFilename) + searchBannerFile = themeSearchImageFile + if os.path.isfile(searchBannerFilename): usersPath = '/users/' + searchNickname followStr += \ From 9f70a96f51388b7e918e10cc105387420ddb7ccc Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 13:19:09 +0000 Subject: [PATCH 08/14] opt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba70efbba..9f7a28c46 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ sudo apt install -y \ In the most common case you'll be using systemd to set up a daemon to run the server. -The following instructions install Epicyon to the **/etc** directory. It's not essential that it be installed there, and it could be in any other preferred directory. +The following instructions install Epicyon to the **/opt** directory. It's not essential that it be installed there, and it could be in any other preferred directory. Add a dedicated user so that we don't have to run as root. From 4c1553e1e767b1f8a169250514cc87cc39845d3b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 14:41:28 +0000 Subject: [PATCH 09/14] Check newswire for no return value --- newsdaemon.py | 6 ++++++ newswire.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/newsdaemon.py b/newsdaemon.py index 9550212bc..2fd69834a 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -471,6 +471,9 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, maxMirroredArticles: int) -> None: """Converts rss items in a newswire into posts """ + if not newswire: + return + basePath = baseDir + '/accounts/news@' + domain + '/outbox' if not os.path.isdir(basePath): os.mkdir(basePath) @@ -669,6 +672,9 @@ def mergeWithPreviousNewswire(oldNewswire: {}, newNewswire: {}) -> None: """Preserve any votes or generated activitypub post filename as rss feeds are updated """ + if not oldNewswire: + return + for published, fields in oldNewswire.items(): if not newNewswire.get(published): continue diff --git a/newswire.py b/newswire.py index cc2eb538e..6cb453b7b 100644 --- a/newswire.py +++ b/newswire.py @@ -354,6 +354,8 @@ def getRSSfromDict(baseDir: str, newswire: {}, rssStr = rss2Header(httpPrefix, None, domainFull, 'Newswire', translate) + if not newswire: + return '' for published, fields in newswire.items(): if '+00:00' in published: published = published.replace('+00:00', 'Z').strip() From 5318beebf049159e0396ab90247de7956e4d5348 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 15:04:33 +0000 Subject: [PATCH 10/14] Check that items are returned --- newswire.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/newswire.py b/newswire.py index 6cb453b7b..069bc8ef7 100644 --- a/newswire.py +++ b/newswire.py @@ -589,8 +589,9 @@ def getDictFromNewswire(session, baseDir: str, domain: str, itemsList = getRSS(baseDir, domain, session, url, moderated, mirrored, maxPostsPerSource, maxFeedSizeKb) - for dateStr, item in itemsList.items(): - result[dateStr] = item + if itemsList: + for dateStr, item in itemsList.items(): + result[dateStr] = item # add blogs from each user account addBlogsToNewswire(baseDir, domain, result, From 896776b54eb83fe2d82ac156a3efa60abcc0b21b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 15:21:42 +0000 Subject: [PATCH 11/14] Increase default maximum rss feed size, but don't overdo it --- epicyon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/epicyon.py b/epicyon.py index 237cdaa82..eacd846ac 100644 --- a/epicyon.py +++ b/epicyon.py @@ -118,7 +118,7 @@ parser.add_argument('--postsPerSource', help='Maximum newswire posts per feed or account') parser.add_argument('--maxFeedSize', dest='maxNewswireFeedSizeKb', type=int, - default=2048, + default=10240, help='Maximum newswire rss/atom feed size in K') parser.add_argument('--maxMirroredArticles', dest='maxMirroredArticles', type=int, From c8858f871a9133b5e125232340e475318851550a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 16:04:25 +0000 Subject: [PATCH 12/14] Check the size of individual rss/atom feed items --- daemon.py | 6 +++++- epicyon.py | 13 ++++++++++++- newsdaemon.py | 3 ++- newswire.py | 34 +++++++++++++++++++++++++--------- tests.py | 6 +++--- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/daemon.py b/daemon.py index f6cad15a3..7587f7005 100644 --- a/daemon.py +++ b/daemon.py @@ -12222,7 +12222,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(publishButtonAtTop: bool, +def runDaemon(maxFeedItemSizeKb: int, + publishButtonAtTop: bool, rssIconAtTop: bool, iconsAsButtons: bool, fullWidthTimelineButtonHeader: bool, @@ -12400,6 +12401,9 @@ def runDaemon(publishButtonAtTop: bool, # above the header image httpd.publishButtonAtTop = publishButtonAtTop + # maximum size of individual RSS feed items, in K + httpd.maxFeedItemSizeKb = maxFeedItemSizeKb + if registration == 'open': httpd.registration = True else: diff --git a/epicyon.py b/epicyon.py index eacd846ac..9bc1144e4 100644 --- a/epicyon.py +++ b/epicyon.py @@ -120,6 +120,11 @@ parser.add_argument('--maxFeedSize', dest='maxNewswireFeedSizeKb', type=int, default=10240, help='Maximum newswire rss/atom feed size in K') +parser.add_argument('--maxFeedItemSizeKb', + dest='maxFeedItemSizeKb', type=int, + default=2048, + help='Maximum size of an individual rss/atom ' + + 'feed item in K') parser.add_argument('--maxMirroredArticles', dest='maxMirroredArticles', type=int, default=100, @@ -2010,6 +2015,11 @@ maxFollowers = \ if maxFollowers is not None: args.maxFollowers = int(maxFollowers) +maxFeedItemSizeKb = \ + getConfigParam(baseDir, 'maxFeedItemSizeKb') +if maxFeedItemSizeKb is not None: + args.maxFeedItemSizeKb = int(maxFeedItemSizeKb) + allowNewsFollowers = \ getConfigParam(baseDir, 'allowNewsFollowers') if allowNewsFollowers is not None: @@ -2053,7 +2063,8 @@ if setTheme(baseDir, themeName, domain): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.publishButtonAtTop, + runDaemon(args.maxFeedItemSizeKb, + args.publishButtonAtTop, args.rssIconAtTop, args.iconsAsButtons, args.fullWidthTimelineButtonHeader, diff --git a/newsdaemon.py b/newsdaemon.py index 2fd69834a..a6e4fc0bd 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -705,7 +705,8 @@ def runNewswireDaemon(baseDir: str, httpd, getDictFromNewswire(httpd.session, baseDir, domain, httpd.maxNewswirePostsPerSource, httpd.maxNewswireFeedSizeKb, - httpd.maxTags) + httpd.maxTags, + httpd.maxFeedItemSizeKb) except Exception as e: print('WARN: unable to update newswire ' + str(e)) time.sleep(120) diff --git a/newswire.py b/newswire.py index 069bc8ef7..3f76e471b 100644 --- a/newswire.py +++ b/newswire.py @@ -126,7 +126,8 @@ def addNewswireDictEntry(baseDir: str, domain: str, def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an xml 2.0 string to a dictionary """ if '' not in xmlStr: @@ -134,7 +135,11 @@ def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, result = {} rssItems = xmlStr.split('') postCtr = 0 + maxBytes = maxFeedItemSizeKb * 1024 for rssItem in rssItems: + if len(rssItem) > maxBytes: + print('WARN: rss feed item is too big') + continue if '' not in rssItem: continue if '' not in rssItem: @@ -205,7 +210,8 @@ def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an atom feed string to a dictionary """ if '' not in xmlStr: @@ -213,7 +219,11 @@ def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, result = {} rssItems = xmlStr.split('') postCtr = 0 + maxBytes = maxFeedItemSizeKb * 1024 for rssItem in rssItems: + if len(rssItem) > maxBytes: + print('WARN: atom feed item is too big') + continue if '' not in rssItem: continue if '' not in rssItem: @@ -283,21 +293,25 @@ def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, def xmlStrToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an xml string to a dictionary """ if 'rss version="2.0"' in xmlStr: return xml2StrToDict(baseDir, domain, - xmlStr, moderated, mirrored, maxPostsPerSource) + xmlStr, moderated, mirrored, + maxPostsPerSource, maxFeedItemSizeKb) elif 'xmlns="http://www.w3.org/2005/Atom"' in xmlStr: return atomFeedToDict(baseDir, domain, - xmlStr, moderated, mirrored, maxPostsPerSource) + xmlStr, moderated, mirrored, + maxPostsPerSource, maxFeedItemSizeKb) return {} def getRSS(baseDir: str, domain: str, session, url: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int, maxFeedSizeKb: int) -> {}: + maxPostsPerSource: int, maxFeedSizeKb: int, + maxFeedItemSizeKb: int) -> {}: """Returns an RSS url as a dict """ if not isinstance(url, str): @@ -325,7 +339,8 @@ def getRSS(baseDir: str, domain: str, session, url: str, not containsInvalidChars(result.text): return xmlStrToDict(baseDir, domain, result.text, moderated, mirrored, - maxPostsPerSource) + maxPostsPerSource, + maxFeedItemSizeKb) else: print('WARN: feed is too large: ' + url) except requests.exceptions.RequestException as e: @@ -549,7 +564,7 @@ def addBlogsToNewswire(baseDir: str, domain: str, newswire: {}, def getDictFromNewswire(session, baseDir: str, domain: str, maxPostsPerSource: int, maxFeedSizeKb: int, - maxTags: int) -> {}: + maxTags: int, maxFeedItemSizeKb: int) -> {}: """Gets rss feeds as a dictionary from newswire file """ subscriptionsFilename = baseDir + '/accounts/newswire.txt' @@ -588,7 +603,8 @@ def getDictFromNewswire(session, baseDir: str, domain: str, itemsList = getRSS(baseDir, domain, session, url, moderated, mirrored, - maxPostsPerSource, maxFeedSizeKb) + maxPostsPerSource, maxFeedSizeKb, + maxFeedItemSizeKb) if itemsList: for dateStr, item in itemsList.items(): result[dateStr] = item diff --git a/tests.py b/tests.py index 647a8376a..c4cf32433 100644 --- a/tests.py +++ b/tests.py @@ -291,7 +291,7 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, @@ -356,7 +356,7 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, @@ -395,7 +395,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, From 0e9bb519859c23bf9c8b641f1b1e0bc5d617e565 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 16:08:31 +0000 Subject: [PATCH 13/14] Re-establish session for newswire if needed --- newsdaemon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/newsdaemon.py b/newsdaemon.py index a6e4fc0bd..ccfa86c7e 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -31,6 +31,7 @@ from utils import saveJson from utils import getStatusNumber from utils import clearFromPostCaches from inbox import storeHashTags +from session import createSession def updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: @@ -694,9 +695,14 @@ def runNewswireDaemon(baseDir: str, httpd, while True: # has the session been created yet? if not httpd.session: - print('Newswire daemon waiting for session') - time.sleep(60) - continue + print('Newswire daemon has no session') + httpd.session = createSession(httpd.proxyType) + if not httpd.session: + print('Newswire daemon waiting for session') + time.sleep(60) + continue + else: + print('Newswire daemon session established') # try to update the feeds newNewswire = None From 2529bce3af15af0713ef3b7175e58d3dd219b645 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 3 Nov 2020 16:10:54 +0000 Subject: [PATCH 14/14] Improve session messages --- newsdaemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newsdaemon.py b/newsdaemon.py index ccfa86c7e..c0a6e8120 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -695,10 +695,10 @@ def runNewswireDaemon(baseDir: str, httpd, while True: # has the session been created yet? if not httpd.session: - print('Newswire daemon has no session') + print('Newswire daemon waiting for session') httpd.session = createSession(httpd.proxyType) if not httpd.session: - print('Newswire daemon waiting for session') + print('Newswire daemon has no session') time.sleep(60) continue else: