diff --git a/daemon.py b/daemon.py index dabaefb56..01cf419b3 100644 --- a/daemon.py +++ b/daemon.py @@ -204,6 +204,8 @@ from devices import E2EEdevicesCollection from devices import E2EEvalidDevice from devices import E2EEaddDevice from newswire import getRSSfromDict +from newswire import runNewswireWatchdog +from newswire import runNewswireDaemon import os @@ -1066,7 +1068,8 @@ class PubServer(BaseHTTPRequestHandler): if self.path.startswith('/icons/') or \ self.path.startswith('/avatars/') or \ - self.path.startswith('/favicon.ico'): + self.path.startswith('/favicon.ico') or \ + self.path.startswith('/newswire.xml'): return False # token based authenticated used by the web interface @@ -5900,7 +5903,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, projectVersion, self._isMinimal(nickname), - YTReplacementDomain) + YTReplacementDomain, + self.server.newswire) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, 'show status done', @@ -6006,7 +6010,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6106,7 +6111,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6206,7 +6212,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6306,7 +6313,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6381,7 +6389,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.allowDeletion, httpPrefix, self.server.projectVersion, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6465,7 +6474,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6568,7 +6578,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6661,7 +6672,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self._isMinimal(nickname), - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6746,7 +6758,8 @@ class PubServer(BaseHTTPRequestHandler): True, httpPrefix, self.server.projectVersion, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.newswire) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -11251,11 +11264,17 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, allowDeletion, debug, maxMentions, maxEmoji, httpd.translate, unitTest, httpd.YTReplacementDomain), daemon=True) + print('Creating scheduled post thread') httpd.thrPostSchedule = \ threadWithTrace(target=runPostSchedule, args=(baseDir, httpd, 20), daemon=True) + print('Creating newswire thread') + httpd.thrNewswireDaemon = \ + threadWithTrace(target=runNewswireDaemon, + args=(baseDir, httpd), daemon=True) + # flags used when restarting the inbox queue httpd.restartInboxQueueInProgress = False httpd.restartInboxQueue = False @@ -11272,6 +11291,12 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, threadWithTrace(target=runPostScheduleWatchdog, args=(projectVersion, httpd), daemon=True) httpd.thrWatchdogSchedule.start() + + print('Creating newswire watchdog') + httpd.thrNewswireWatchdog = \ + threadWithTrace(target=runNewswireWatchdog, + args=(projectVersion, httpd), daemon=True) + httpd.thrNewswireWatchdog.start() else: httpd.thrInboxQueue.start() httpd.thrPostSchedule.start() diff --git a/epicyon-profile.css b/epicyon-profile.css index eac77ae78..ebf9d99a4 100644 --- a/epicyon-profile.css +++ b/epicyon-profile.css @@ -14,6 +14,7 @@ --main-header-color-roles: #282237; --main-fg-color: #dddddd; --column-left-fg-color: #dddddd; + --column-right-fg-color: #dddddd; --main-link-color: #999; --main-link-color-hover: #bbb; --main-visited-color: #888; @@ -24,6 +25,7 @@ --font-color-header: #ccc; --font-size-button-mobile: 34px; --font-size-links: 18px; + --font-size-newswire: 18px; --font-size: 30px; --font-size2: 24px; --font-size3: 38px; @@ -64,6 +66,7 @@ --quote-font-weight: normal; --quote-font-size: 120%; --line-spacing: 130%; + --line-spacing-newswire: 100%; --column-left-width: 10vw; --column-center-width: 80vw; --column-right-width: 10vw; @@ -72,6 +75,7 @@ --column-left-header-size: 20px; --column-left-icon-size: 20%; --column-right-icon-size: 20%; + --newswire-date-color: white; } @font-face { @@ -221,6 +225,18 @@ a:focus { width: 50%; } +.newswireItem { + font-size: var(--font-size-newswire); + color: var(--column-right-fg-color); + line-height: var(--line-spacing-newswire); +} + +.newswireDate { + font-size: var(--font-size-newswire); + color: var(--newswire-date-color); + float: right; +} + .new-post-text { font-size: var(--font-size2); font-family: Arial, Helvetica, sans-serif; @@ -1007,6 +1023,7 @@ aside .toggle-inside li { .col-right { background-color: var(--column-left-color); color: var(--column-left-fg-color); + padding-left: 10px; padding-right: 30px; font-size: var(--font-size-links); width: var(--column-right-width); diff --git a/newswire.py b/newswire.py index d295b1273..111ad6bf7 100644 --- a/newswire.py +++ b/newswire.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +import time import requests from socket import error as SocketError import errno @@ -144,10 +145,16 @@ def getRSSfromDict(baseDir: str, newswire: {}, None, domainFull, 'Newswire', translate) for published, fields in newswire.items(): + published = published.replace('+00:00', 'Z').strip() + published = published.replace(' ', 'T') + try: + pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") + except BaseException: + continue rssStr += '\n' rssStr += ' ' + fields[0] + '\n' rssStr += ' ' + fields[1] + '\n' - pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ") + rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") rssStr += ' ' + rssDateStr + '\n' rssStr += '\n' @@ -172,6 +179,52 @@ def getDictFromNewswire(session, baseDir: str) -> {}: continue if url.startswith('#'): continue - result = dict(result.items() + getRSS(session, url).items()) - sortedResult = OrderedDict(sorted(result.items(), reverse=False)) + itemsList = getRSS(session, url) + for dateStr, item in itemsList.items(): + result[dateStr] = item + sortedResult = OrderedDict(sorted(result.items(), reverse=True)) return sortedResult + + +def runNewswireDaemon(baseDir: str, httpd): + """Periodically updates RSS feeds + """ + # initial sleep to allow the system to start up + time.sleep(70) + while True: + # has the session been created yet? + if not httpd.session: + print('Newswire daemon waiting for session') + time.sleep(60) + continue + + # try to update the feeds + newNewswire = None + try: + newNewswire = getDictFromNewswire(httpd.session, baseDir) + except BaseException: + print('WARN: unable to update newswire') + time.sleep(120) + continue + + httpd.newswire = newNewswire + print('Newswire updated') + # wait a while before the next feeds update + time.sleep(1200) + + +def runNewswireWatchdog(projectVersion: str, httpd) -> None: + """This tries to keep the newswire update thread running even if it dies + """ + print('Starting newswire watchdog') + newswireOriginal = \ + httpd.thrPostSchedule.clone(runNewswireDaemon) + httpd.thrNewswireDaemon.start() + while True: + time.sleep(50) + if not httpd.thrNewswireDaemon.isAlive(): + httpd.thrNewswireDaemon.kill() + httpd.thrNewswireDaemon = \ + newswireOriginal.clone(runNewswireDaemon) + httpd.thrNewswireDaemon.start() + print('Restarting newswire daemon...') diff --git a/webinterface.py b/webinterface.py index a867e38c6..b381ae3af 100644 --- a/webinterface.py +++ b/webinterface.py @@ -5337,9 +5337,22 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, return htmlStr +def htmlNewswire(newswire: str) -> str: + """Converts a newswire dict into html + """ + htmlStr = '' + for dateStr, item in newswire.items(): + htmlStr += '

' + \ + '' + item[0] + '' + htmlStr += '

' + return htmlStr + + def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, httpPrefix: str, translate: {}, - iconsDir: str, moderator: bool) -> str: + iconsDir: str, moderator: bool, + newswire: {}) -> str: """Returns html content for the right column """ htmlStr = '' @@ -5400,6 +5413,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, else: htmlStr += '
\n' + htmlStr += htmlNewswire(newswire) return htmlStr @@ -5413,7 +5427,8 @@ def htmlTimeline(defaultTimeline: str, httpPrefix: str, projectVersion: str, manuallyApproveFollowers: bool, minimal: bool, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + newswire: {}) -> str: """Show the timeline as html """ timelineStartTime = time.time() @@ -6021,7 +6036,7 @@ def htmlTimeline(defaultTimeline: str, # right column rightColumnStr = getRightColumnContent(baseDir, nickname, domainFull, httpPrefix, translate, iconsDir, - moderator) + moderator, newswire) tlStr += ' ' + \ rightColumnStr + ' \n' tlStr += ' \n' @@ -6062,7 +6077,8 @@ def htmlShares(defaultTimeline: str, nickname: str, domain: str, port: int, allowDeletion: bool, httpPrefix: str, projectVersion: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + newswire: {}) -> str: """Show the shares timeline as html """ manuallyApproveFollowers = \ @@ -6074,7 +6090,7 @@ def htmlShares(defaultTimeline: str, nickname, domain, port, None, 'tlshares', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - False, YTReplacementDomain) + False, YTReplacementDomain, newswire) def htmlInbox(defaultTimeline: str, @@ -6084,7 +6100,8 @@ def htmlInbox(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the inbox as html """ manuallyApproveFollowers = \ @@ -6096,7 +6113,7 @@ def htmlInbox(defaultTimeline: str, nickname, domain, port, inboxJson, 'inbox', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlBookmarks(defaultTimeline: str, @@ -6106,7 +6123,8 @@ def htmlBookmarks(defaultTimeline: str, nickname: str, domain: str, port: int, bookmarksJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the bookmarks as html """ manuallyApproveFollowers = \ @@ -6118,7 +6136,7 @@ def htmlBookmarks(defaultTimeline: str, nickname, domain, port, bookmarksJson, 'tlbookmarks', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlEvents(defaultTimeline: str, @@ -6128,7 +6146,8 @@ def htmlEvents(defaultTimeline: str, nickname: str, domain: str, port: int, bookmarksJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the events as html """ manuallyApproveFollowers = \ @@ -6140,7 +6159,7 @@ def htmlEvents(defaultTimeline: str, nickname, domain, port, bookmarksJson, 'tlevents', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlInboxDMs(defaultTimeline: str, @@ -6150,7 +6169,8 @@ def htmlInboxDMs(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the DM timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6158,7 +6178,7 @@ def htmlInboxDMs(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'dm', allowDeletion, httpPrefix, projectVersion, False, minimal, - YTReplacementDomain) + YTReplacementDomain, newswire) def htmlInboxReplies(defaultTimeline: str, @@ -6168,7 +6188,8 @@ def htmlInboxReplies(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the replies timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6176,7 +6197,7 @@ def htmlInboxReplies(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlreplies', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlInboxMedia(defaultTimeline: str, @@ -6186,7 +6207,8 @@ def htmlInboxMedia(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the media timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6194,7 +6216,7 @@ def htmlInboxMedia(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlmedia', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlInboxBlogs(defaultTimeline: str, @@ -6204,7 +6226,8 @@ def htmlInboxBlogs(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the blogs timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6212,7 +6235,7 @@ def htmlInboxBlogs(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlblogs', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain) + minimal, YTReplacementDomain, newswire) def htmlModeration(defaultTimeline: str, @@ -6222,7 +6245,8 @@ def htmlModeration(defaultTimeline: str, nickname: str, domain: str, port: int, inboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + newswire: {}) -> str: """Show the moderation feed as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6230,7 +6254,7 @@ def htmlModeration(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'moderation', allowDeletion, httpPrefix, projectVersion, True, False, - YTReplacementDomain) + YTReplacementDomain, newswire) def htmlOutbox(defaultTimeline: str, @@ -6240,7 +6264,8 @@ def htmlOutbox(defaultTimeline: str, nickname: str, domain: str, port: int, outboxJson: {}, allowDeletion: bool, httpPrefix: str, projectVersion: str, - minimal: bool, YTReplacementDomain: str) -> str: + minimal: bool, YTReplacementDomain: str, + newswire: {}) -> str: """Show the Outbox as html """ manuallyApproveFollowers = \ @@ -6251,7 +6276,7 @@ def htmlOutbox(defaultTimeline: str, nickname, domain, port, outboxJson, 'outbox', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, minimal, - YTReplacementDomain) + YTReplacementDomain, newswire) def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int,