diff --git a/README.md b/README.md index 52d90d0b6..9d530e8f7 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ sudo apt install -y \ In the most common case you'll be using systemd to set up a daemon to run the server. -The following instructions install Epicyon to the **/etc** directory. It's not essential that it be installed there, and it could be in any other preferred directory. +The following instructions install Epicyon to the **/opt** directory. It's not essential that it be installed there, and it could be in any other preferred directory. Add a dedicated user so that we don't have to run as root. diff --git a/daemon.py b/daemon.py index f6cad15a3..7587f7005 100644 --- a/daemon.py +++ b/daemon.py @@ -12222,7 +12222,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(publishButtonAtTop: bool, +def runDaemon(maxFeedItemSizeKb: int, + publishButtonAtTop: bool, rssIconAtTop: bool, iconsAsButtons: bool, fullWidthTimelineButtonHeader: bool, @@ -12400,6 +12401,9 @@ def runDaemon(publishButtonAtTop: bool, # above the header image httpd.publishButtonAtTop = publishButtonAtTop + # maximum size of individual RSS feed items, in K + httpd.maxFeedItemSizeKb = maxFeedItemSizeKb + if registration == 'open': httpd.registration = True else: diff --git a/epicyon.py b/epicyon.py index 237cdaa82..9bc1144e4 100644 --- a/epicyon.py +++ b/epicyon.py @@ -118,8 +118,13 @@ parser.add_argument('--postsPerSource', help='Maximum newswire posts per feed or account') parser.add_argument('--maxFeedSize', dest='maxNewswireFeedSizeKb', type=int, - default=2048, + default=10240, help='Maximum newswire rss/atom feed size in K') +parser.add_argument('--maxFeedItemSizeKb', + dest='maxFeedItemSizeKb', type=int, + default=2048, + help='Maximum size of an individual rss/atom ' + + 'feed item in K') parser.add_argument('--maxMirroredArticles', dest='maxMirroredArticles', type=int, default=100, @@ -2010,6 +2015,11 @@ maxFollowers = \ if maxFollowers is not None: args.maxFollowers = int(maxFollowers) +maxFeedItemSizeKb = \ + getConfigParam(baseDir, 'maxFeedItemSizeKb') +if maxFeedItemSizeKb is not None: + args.maxFeedItemSizeKb = int(maxFeedItemSizeKb) + allowNewsFollowers = \ getConfigParam(baseDir, 'allowNewsFollowers') if allowNewsFollowers is not None: @@ -2053,7 +2063,8 @@ if setTheme(baseDir, themeName, domain): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.publishButtonAtTop, + runDaemon(args.maxFeedItemSizeKb, + args.publishButtonAtTop, args.rssIconAtTop, args.iconsAsButtons, args.fullWidthTimelineButtonHeader, diff --git a/newsdaemon.py b/newsdaemon.py index 9550212bc..c0a6e8120 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -31,6 +31,7 @@ from utils import saveJson from utils import getStatusNumber from utils import clearFromPostCaches from inbox import storeHashTags +from session import createSession def updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: @@ -471,6 +472,9 @@ def convertRSStoActivityPub(baseDir: str, httpPrefix: str, maxMirroredArticles: int) -> None: """Converts rss items in a newswire into posts """ + if not newswire: + return + basePath = baseDir + '/accounts/news@' + domain + '/outbox' if not os.path.isdir(basePath): os.mkdir(basePath) @@ -669,6 +673,9 @@ def mergeWithPreviousNewswire(oldNewswire: {}, newNewswire: {}) -> None: """Preserve any votes or generated activitypub post filename as rss feeds are updated """ + if not oldNewswire: + return + for published, fields in oldNewswire.items(): if not newNewswire.get(published): continue @@ -689,8 +696,13 @@ def runNewswireDaemon(baseDir: str, httpd, # has the session been created yet? if not httpd.session: print('Newswire daemon waiting for session') - time.sleep(60) - continue + httpd.session = createSession(httpd.proxyType) + if not httpd.session: + print('Newswire daemon has no session') + time.sleep(60) + continue + else: + print('Newswire daemon session established') # try to update the feeds newNewswire = None @@ -699,7 +711,8 @@ def runNewswireDaemon(baseDir: str, httpd, getDictFromNewswire(httpd.session, baseDir, domain, httpd.maxNewswirePostsPerSource, httpd.maxNewswireFeedSizeKb, - httpd.maxTags) + httpd.maxTags, + httpd.maxFeedItemSizeKb) except Exception as e: print('WARN: unable to update newswire ' + str(e)) time.sleep(120) diff --git a/newswire.py b/newswire.py index cc2eb538e..3f76e471b 100644 --- a/newswire.py +++ b/newswire.py @@ -126,7 +126,8 @@ def addNewswireDictEntry(baseDir: str, domain: str, def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an xml 2.0 string to a dictionary """ if '' not in xmlStr: @@ -134,7 +135,11 @@ def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, result = {} rssItems = xmlStr.split('') postCtr = 0 + maxBytes = maxFeedItemSizeKb * 1024 for rssItem in rssItems: + if len(rssItem) > maxBytes: + print('WARN: rss feed item is too big') + continue if '' not in rssItem: continue if '' not in rssItem: @@ -205,7 +210,8 @@ def xml2StrToDict(baseDir: str, domain: str, xmlStr: str, def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an atom feed string to a dictionary """ if '' not in xmlStr: @@ -213,7 +219,11 @@ def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, result = {} rssItems = xmlStr.split('') postCtr = 0 + maxBytes = maxFeedItemSizeKb * 1024 for rssItem in rssItems: + if len(rssItem) > maxBytes: + print('WARN: atom feed item is too big') + continue if '' not in rssItem: continue if '' not in rssItem: @@ -283,21 +293,25 @@ def atomFeedToDict(baseDir: str, domain: str, xmlStr: str, def xmlStrToDict(baseDir: str, domain: str, xmlStr: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int) -> {}: + maxPostsPerSource: int, + maxFeedItemSizeKb: int) -> {}: """Converts an xml string to a dictionary """ if 'rss version="2.0"' in xmlStr: return xml2StrToDict(baseDir, domain, - xmlStr, moderated, mirrored, maxPostsPerSource) + xmlStr, moderated, mirrored, + maxPostsPerSource, maxFeedItemSizeKb) elif 'xmlns="http://www.w3.org/2005/Atom"' in xmlStr: return atomFeedToDict(baseDir, domain, - xmlStr, moderated, mirrored, maxPostsPerSource) + xmlStr, moderated, mirrored, + maxPostsPerSource, maxFeedItemSizeKb) return {} def getRSS(baseDir: str, domain: str, session, url: str, moderated: bool, mirrored: bool, - maxPostsPerSource: int, maxFeedSizeKb: int) -> {}: + maxPostsPerSource: int, maxFeedSizeKb: int, + maxFeedItemSizeKb: int) -> {}: """Returns an RSS url as a dict """ if not isinstance(url, str): @@ -325,7 +339,8 @@ def getRSS(baseDir: str, domain: str, session, url: str, not containsInvalidChars(result.text): return xmlStrToDict(baseDir, domain, result.text, moderated, mirrored, - maxPostsPerSource) + maxPostsPerSource, + maxFeedItemSizeKb) else: print('WARN: feed is too large: ' + url) except requests.exceptions.RequestException as e: @@ -354,6 +369,8 @@ def getRSSfromDict(baseDir: str, newswire: {}, rssStr = rss2Header(httpPrefix, None, domainFull, 'Newswire', translate) + if not newswire: + return '' for published, fields in newswire.items(): if '+00:00' in published: published = published.replace('+00:00', 'Z').strip() @@ -547,7 +564,7 @@ def addBlogsToNewswire(baseDir: str, domain: str, newswire: {}, def getDictFromNewswire(session, baseDir: str, domain: str, maxPostsPerSource: int, maxFeedSizeKb: int, - maxTags: int) -> {}: + maxTags: int, maxFeedItemSizeKb: int) -> {}: """Gets rss feeds as a dictionary from newswire file """ subscriptionsFilename = baseDir + '/accounts/newswire.txt' @@ -586,9 +603,11 @@ def getDictFromNewswire(session, baseDir: str, domain: str, itemsList = getRSS(baseDir, domain, session, url, moderated, mirrored, - maxPostsPerSource, maxFeedSizeKb) - for dateStr, item in itemsList.items(): - result[dateStr] = item + maxPostsPerSource, maxFeedSizeKb, + maxFeedItemSizeKb) + if itemsList: + for dateStr, item in itemsList.items(): + result[dateStr] = item # add blogs from each user account addBlogsToNewswire(baseDir, domain, result, diff --git a/tests.py b/tests.py index 647a8376a..c4cf32433 100644 --- a/tests.py +++ b/tests.py @@ -291,7 +291,7 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, @@ -356,7 +356,7 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, @@ -395,7 +395,7 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(False, True, False, False, True, 10, False, + runDaemon(2048, False, True, False, False, True, 10, False, 0, 100, 1024, 5, False, 0, False, 1, False, False, False, 5, True, True, 'en', __version__, diff --git a/webinterface.py b/webinterface.py index 84b1550b6..9509450b0 100644 --- a/webinterface.py +++ b/webinterface.py @@ -3557,9 +3557,10 @@ def htmlProfile(rssIconAtTop: bool, # If this is the news account then show a different banner if isSystemAccount(nickname): + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) profileHeaderStr = \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' if loginButton: profileHeaderStr += '
' + loginButton + '
\n' @@ -5604,20 +5605,24 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, editImageClass = '' if showHeaderImage: - leftColumnImageFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + \ - '/left_col_image.png' + leftImageFile, leftColumnImageFilename = \ + getLeftImageFile(baseDir, nickname, domain) if not os.path.isfile(leftColumnImageFilename): theme = getConfigParam(baseDir, 'theme').lower() if theme == 'default': theme = '' else: theme = '_' + theme - themeLeftColumnImageFilename = \ - baseDir + '/img/left_col_image' + theme + '.png' + themeLeftImageFile, themeLeftColumnImageFilename = \ + getImageFile(baseDir, 'left_col_image', baseDir + '/img', + nickname, domain) if os.path.isfile(themeLeftColumnImageFilename): + leftColumnImageFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/' + themeLeftImageFile copyfile(themeLeftColumnImageFilename, leftColumnImageFilename) + leftImageFile = themeLeftImageFile # show the image at the top of the column editImageClass = 'leftColEdit' @@ -5627,7 +5632,7 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, '\n
\n' + \ ' \n' + \ + nickname + '/' + leftImageFile + '" />\n' + \ '
\n' if showBackButton: @@ -5858,20 +5863,24 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, # show a column header image, eg. title of the theme or newswire banner editImageClass = '' if showHeaderImage: - rightColumnImageFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + \ - '/right_col_image.png' + rightImageFile, rightColumnImageFilename = \ + getRightImageFile(baseDir, nickname, domain) if not os.path.isfile(rightColumnImageFilename): theme = getConfigParam(baseDir, 'theme').lower() if theme == 'default': theme = '' else: theme = '_' + theme - themeRightColumnImageFilename = \ - baseDir + '/img/right_col_image' + theme + '.png' + themeRightImageFile, themeRightColumnImageFilename = \ + getImageFile(baseDir, 'right_col_image', baseDir + '/img', + nickname, domain) if os.path.isfile(themeRightColumnImageFilename): + rightColumnImageFilename = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + '/' + themeRightImageFile copyfile(themeRightColumnImageFilename, rightColumnImageFilename) + rightImageFile = themeRightImageFile # show the image at the top of the column editImageClass = 'rightColEdit' @@ -5881,7 +5890,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, '\n
\n' + \ ' \n' + \ + nickname + '/' + rightImageFile + '" />\n' + \ '
\n' if (showPublishButton or editor or rssIconAtTop) and not showHeaderImage: @@ -6000,11 +6009,16 @@ def htmlLinksMobile(cssCache: {}, baseDir: str, else: editor = isEditor(baseDir, nickname) + domain = domainFull + if ':' in domain: + domain = domain.split(':')[0] + htmlStr = htmlHeader(cssFilename, profileStyle) + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) htmlStr += \ '' + \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' htmlStr += '
' + \ headerButtonsFrontScreen(translate, nickname, @@ -6063,10 +6077,12 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, showPublishButton = editor htmlStr = htmlHeader(cssFilename, profileStyle) + + bannerFile, bannerFilename = getBannerFile(baseDir, nickname, domain) htmlStr += \ '' + \ '\n' + 'src="/users/' + nickname + '/' + bannerFile + '" />\n' htmlStr += '
' + \ headerButtonsFrontScreen(translate, nickname, @@ -6084,37 +6100,48 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, return htmlStr -def getBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): +def getImageFile(baseDir: str, name: str, directory: str, + nickname: str, domain: str) -> (str, str): """ - returns the banner filename + returns the filenames for an image with the given name """ bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') bannerFile = '' bannerFilename = '' for ext in bannerExtensions: - bannerFile = 'banner.' + ext - bannerFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/' + bannerFile + bannerFile = name + '.' + ext + bannerFilename = directory + '/' + bannerFile if os.path.isfile(bannerFilename): break return bannerFile, bannerFilename +def getBannerFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + def getSearchBannerFile(baseDir: str, nickname: str, domain: str) -> (str, str): - """ - returns the search banner filename - """ - bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') - bannerFile = '' - bannerFilename = '' - for ext in bannerExtensions: - bannerFile = 'search_banner.' + ext - bannerFilename = baseDir + '/accounts/' + \ - nickname + '@' + domain + '/' + bannerFile - if os.path.isfile(bannerFilename): - break - return bannerFile, bannerFilename + return getImageFile(baseDir, 'search_banner', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + +def getLeftImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'left_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) + + +def getRightImageFile(baseDir: str, + nickname: str, domain: str) -> (str, str): + return getImageFile(baseDir, 'right_col_image', + baseDir + '/accounts/' + nickname + '@' + domain, + nickname, domain) def headerButtonsFrontScreen(translate: {}, @@ -8698,17 +8725,17 @@ def htmlSearch(cssCache: {}, translate: {}, theme = '' else: theme = '_' + theme - bannerExtensions = ('png', 'jpg', 'jpeg', 'gif', 'avif', 'webp') - for ext in bannerExtensions: - searchBannerFile = 'search_banner.' + ext + themeSearchImageFile, themeSearchBannerFilename = \ + getImageFile(baseDir, 'search_banner', baseDir + '/img', + searchNickname, domain) + if os.path.isfile(themeSearchBannerFilename): searchBannerFilename = \ baseDir + '/accounts/' + \ - searchNickname + '@' + domain + '/' + searchBannerFile - themeSearchBannerFilename = \ - baseDir + '/img/search_banner' + theme + '.' + ext - if os.path.isfile(themeSearchBannerFilename): - copyfile(themeSearchBannerFilename, searchBannerFilename) - break + searchNickname + '@' + domain + '/' + themeSearchImageFile + copyfile(themeSearchBannerFilename, + searchBannerFilename) + searchBannerFile = themeSearchImageFile + if os.path.isfile(searchBannerFilename): usersPath = '/users/' + searchNickname followStr += \