Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon into main

daemon.py

@@ -204,6 +204,8 @@ from devices import E2EEdevicesCollection
 from devices import E2EEvalidDevice
 from devices import E2EEaddDevice
 from newswire import getRSSfromDict
+from newswire import runNewswireWatchdog
+from newswire import runNewswireDaemon
 import os
 
 
@@ -1066,7 +1068,8 @@ class PubServer(BaseHTTPRequestHandler):
 
         if self.path.startswith('/icons/') or \
            self.path.startswith('/avatars/') or \
-           self.path.startswith('/favicon.ico'):
+           self.path.startswith('/favicon.ico') or \
+           self.path.startswith('/newswire.xml'):
             return False
 
         # token based authenticated used by the web interface
@@ -5900,7 +5903,8 @@ class PubServer(BaseHTTPRequestHandler):
                                         httpPrefix,
                                         projectVersion,
                                         self._isMinimal(nickname),
-                                        YTReplacementDomain)
+                                        YTReplacementDomain,
+                                        self.server.newswire)
                         if GETstartTime:
                             self._benchmarkGETtimings(GETstartTime, GETtimings,
                                                       'show status done',
@@ -6006,7 +6010,8 @@ class PubServer(BaseHTTPRequestHandler):
                                          httpPrefix,
                                          self.server.projectVersion,
                                          self._isMinimal(nickname),
-                                         self.server.YTReplacementDomain)
+                                         self.server.YTReplacementDomain,
+                                         self.server.newswire)
                         msg = msg.encode('utf-8')
                         self._set_headers('text/html', len(msg),
                                           cookie, callingDomain)
@@ -6106,7 +6111,8 @@ class PubServer(BaseHTTPRequestHandler):
                                          httpPrefix,
                                          self.server.projectVersion,
                                          self._isMinimal(nickname),
-                                         self.server.YTReplacementDomain)
+                                         self.server.YTReplacementDomain,
+                                         self.server.newswire)
                     msg = msg.encode('utf-8')
                     self._set_headers('text/html', len(msg),
                                       cookie, callingDomain)
@@ -6206,7 +6212,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        httpPrefix,
                                        self.server.projectVersion,
                                        self._isMinimal(nickname),
-                                       self.server.YTReplacementDomain)
+                                       self.server.YTReplacementDomain,
+                                       self.server.newswire)
                     msg = msg.encode('utf-8')
                     self._set_headers('text/html', len(msg),
                                       cookie, callingDomain)
@@ -6306,7 +6313,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        httpPrefix,
                                        self.server.projectVersion,
                                        self._isMinimal(nickname),
-                                       self.server.YTReplacementDomain)
+                                       self.server.YTReplacementDomain,
+                                       self.server.newswire)
                     msg = msg.encode('utf-8')
                     self._set_headers('text/html', len(msg),
                                       cookie, callingDomain)
@@ -6381,7 +6389,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.allowDeletion,
                                    httpPrefix,
                                    self.server.projectVersion,
-                                   self.server.YTReplacementDomain)
+                                   self.server.YTReplacementDomain,
+                                   self.server.newswire)
                     msg = msg.encode('utf-8')
                     self._set_headers('text/html', len(msg),
                                       cookie, callingDomain)
@@ -6465,7 +6474,8 @@ class PubServer(BaseHTTPRequestHandler):
                                           httpPrefix,
                                           self.server.projectVersion,
                                           self._isMinimal(nickname),
-                                          self.server.YTReplacementDomain)
+                                          self.server.YTReplacementDomain,
+                                          self.server.newswire)
                         msg = msg.encode('utf-8')
                         self._set_headers('text/html', len(msg),
                                           cookie, callingDomain)
@@ -6568,7 +6578,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        httpPrefix,
                                        self.server.projectVersion,
                                        self._isMinimal(nickname),
-                                       self.server.YTReplacementDomain)
+                                       self.server.YTReplacementDomain,
+                                       self.server.newswire)
                         msg = msg.encode('utf-8')
                         self._set_headers('text/html', len(msg),
                                           cookie, callingDomain)
@@ -6661,7 +6672,8 @@ class PubServer(BaseHTTPRequestHandler):
                                httpPrefix,
                                self.server.projectVersion,
                                self._isMinimal(nickname),
-                               self.server.YTReplacementDomain)
+                               self.server.YTReplacementDomain,
+                               self.server.newswire)
                 msg = msg.encode('utf-8')
                 self._set_headers('text/html', len(msg),
                                   cookie, callingDomain)
@@ -6746,7 +6758,8 @@ class PubServer(BaseHTTPRequestHandler):
                                            True,
                                            httpPrefix,
                                            self.server.projectVersion,
-                                           self.server.YTReplacementDomain)
+                                           self.server.YTReplacementDomain,
+                                           self.server.newswire)
                         msg = msg.encode('utf-8')
                         self._set_headers('text/html', len(msg),
                                           cookie, callingDomain)
@@ -11251,11 +11264,17 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool,
                               allowDeletion, debug, maxMentions, maxEmoji,
                               httpd.translate, unitTest,
                               httpd.YTReplacementDomain), daemon=True)
+
     print('Creating scheduled post thread')
     httpd.thrPostSchedule = \
         threadWithTrace(target=runPostSchedule,
                         args=(baseDir, httpd, 20), daemon=True)
 
+    print('Creating newswire thread')
+    httpd.thrNewswireDaemon = \
+        threadWithTrace(target=runNewswireDaemon,
+                        args=(baseDir, httpd), daemon=True)
+
     # flags used when restarting the inbox queue
     httpd.restartInboxQueueInProgress = False
     httpd.restartInboxQueue = False
@@ -11272,6 +11291,12 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool,
             threadWithTrace(target=runPostScheduleWatchdog,
                             args=(projectVersion, httpd), daemon=True)
         httpd.thrWatchdogSchedule.start()
+
+        print('Creating newswire watchdog')
+        httpd.thrNewswireWatchdog = \
+            threadWithTrace(target=runNewswireWatchdog,
+                            args=(projectVersion, httpd), daemon=True)
+        httpd.thrNewswireWatchdog.start()
     else:
         httpd.thrInboxQueue.start()
         httpd.thrPostSchedule.start()

epicyon-profile.css

@@ -14,6 +14,7 @@
     --main-header-color-roles: #282237;
     --main-fg-color: #dddddd;
     --column-left-fg-color: #dddddd;
+    --column-right-fg-color: #dddddd;
     --main-link-color: #999;
     --main-link-color-hover: #bbb;
     --main-visited-color: #888;
@@ -24,6 +25,7 @@
     --font-color-header: #ccc;
     --font-size-button-mobile: 34px;
     --font-size-links: 18px;
+    --font-size-newswire: 18px;
     --font-size: 30px;
     --font-size2: 24px;
     --font-size3: 38px;
@@ -64,6 +66,7 @@
     --quote-font-weight: normal;
     --quote-font-size: 120%;
     --line-spacing: 130%;
+    --line-spacing-newswire: 100%;
     --column-left-width: 10vw;
     --column-center-width: 80vw;
     --column-right-width: 10vw;
@@ -72,6 +75,7 @@
     --column-left-header-size: 20px;
     --column-left-icon-size: 20%;
     --column-right-icon-size: 20%;
+    --newswire-date-color: white;
 }
 
 @font-face {
@@ -221,6 +225,18 @@ a:focus {
     width: 50%;
 }
 
+.newswireItem {
+    font-size: var(--font-size-newswire);
+    color: var(--column-right-fg-color);
+    line-height: var(--line-spacing-newswire);
+}
+
+.newswireDate {
+    font-size: var(--font-size-newswire);
+    color: var(--newswire-date-color);
+    float: right;
+}
+
 .new-post-text {
     font-size: var(--font-size2);
     font-family: Arial, Helvetica, sans-serif;
@@ -1007,6 +1023,7 @@ aside .toggle-inside li {
     .col-right {
         background-color: var(--column-left-color);
         color: var(--column-left-fg-color);
+        padding-left: 10px;
         padding-right: 30px;
         font-size: var(--font-size-links);
         width: var(--column-right-width);

newswire.py

@@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import os
+import time
 import requests
 from socket import error as SocketError
 import errno
@@ -144,10 +145,16 @@ def getRSSfromDict(baseDir: str, newswire: {},
                         None, domainFull,
                         'Newswire', translate)
     for published, fields in newswire.items():
+        published = published.replace('+00:00', 'Z').strip()
+        published = published.replace(' ', 'T')
+        try:
+            pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
+        except BaseException:
+            continue
         rssStr += '<item>\n'
         rssStr += '  <title>' + fields[0] + '</title>\n'
         rssStr += '  <link>' + fields[1] + '</link>\n'
-        pubDate = datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
+
         rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
         rssStr += '  <pubDate>' + rssDateStr + '</pubDate>\n'
         rssStr += '</item>\n'
@@ -172,6 +179,52 @@ def getDictFromNewswire(session, baseDir: str) -> {}:
             continue
         if url.startswith('#'):
             continue
-        result = dict(result.items() + getRSS(session, url).items())
+        itemsList = getRSS(session, url)
-    sortedResult = OrderedDict(sorted(result.items(), reverse=False))
+        for dateStr, item in itemsList.items():
+            result[dateStr] = item
+    sortedResult = OrderedDict(sorted(result.items(), reverse=True))
     return sortedResult
+
+
+def runNewswireDaemon(baseDir: str, httpd):
+    """Periodically updates RSS feeds
+    """
+    # initial sleep to allow the system to start up
+    time.sleep(70)
+    while True:
+        # has the session been created yet?
+        if not httpd.session:
+            print('Newswire daemon waiting for session')
+            time.sleep(60)
+            continue
+
+        # try to update the feeds
+        newNewswire = None
+        try:
+            newNewswire = getDictFromNewswire(httpd.session, baseDir)
+        except BaseException:
+            print('WARN: unable to update newswire')
+            time.sleep(120)
+            continue
+
+        httpd.newswire = newNewswire
+        print('Newswire updated')
+        # wait a while before the next feeds update
+        time.sleep(1200)
+
+
+def runNewswireWatchdog(projectVersion: str, httpd) -> None:
+    """This tries to keep the newswire update thread running even if it dies
+    """
+    print('Starting newswire watchdog')
+    newswireOriginal = \
+        httpd.thrPostSchedule.clone(runNewswireDaemon)
+    httpd.thrNewswireDaemon.start()
+    while True:
+        time.sleep(50)
+        if not httpd.thrNewswireDaemon.isAlive():
+            httpd.thrNewswireDaemon.kill()
+            httpd.thrNewswireDaemon = \
+                newswireOriginal.clone(runNewswireDaemon)
+            httpd.thrNewswireDaemon.start()
+            print('Restarting newswire daemon...')

webinterface.py

@@ -5337,9 +5337,22 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str,
     return htmlStr
 
 
+def htmlNewswire(newswire: str) -> str:
+    """Converts a newswire dict into html
+    """
+    htmlStr = ''
+    for dateStr, item in newswire.items():
+        htmlStr += '<p class="newswireItem">' + \
+            '<a href="' + item[1] + '">' + item[0] + '</a>'
+        htmlStr += ' <label class="newswireDate">'
+        htmlStr += dateStr.replace('+00:00', '') + '</label></p>'
+    return htmlStr
+
+
 def getRightColumnContent(baseDir: str, nickname: str, domainFull: str,
                           httpPrefix: str, translate: {},
-                          iconsDir: str, moderator: bool) -> str:
+                          iconsDir: str, moderator: bool,
+                          newswire: {}) -> str:
     """Returns html content for the right column
     """
     htmlStr = ''
@@ -5400,6 +5413,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str,
     else:
         htmlStr += '      <br>\n'
 
+    htmlStr += htmlNewswire(newswire)
     return htmlStr
 
 
@@ -5413,7 +5427,8 @@ def htmlTimeline(defaultTimeline: str,
                  httpPrefix: str, projectVersion: str,
                  manuallyApproveFollowers: bool,
                  minimal: bool,
-                 YTReplacementDomain: str) -> str:
+                 YTReplacementDomain: str,
+                 newswire: {}) -> str:
     """Show the timeline as html
     """
     timelineStartTime = time.time()
@@ -6021,7 +6036,7 @@ def htmlTimeline(defaultTimeline: str,
     # right column
     rightColumnStr = getRightColumnContent(baseDir, nickname, domainFull,
                                            httpPrefix, translate, iconsDir,
-                                           moderator)
+                                           moderator, newswire)
     tlStr += '  <td valign="top" class="col-right">' + \
         rightColumnStr + '  </td>\n'
     tlStr += '  </tr>\n'
@@ -6062,7 +6077,8 @@ def htmlShares(defaultTimeline: str,
                nickname: str, domain: str, port: int,
                allowDeletion: bool,
                httpPrefix: str, projectVersion: str,
-               YTReplacementDomain: str) -> str:
+               YTReplacementDomain: str,
+               newswire: {}) -> str:
     """Show the shares timeline as html
     """
     manuallyApproveFollowers = \
@@ -6074,7 +6090,7 @@ def htmlShares(defaultTimeline: str,
                         nickname, domain, port, None,
                         'tlshares', allowDeletion,
                         httpPrefix, projectVersion, manuallyApproveFollowers,
-                        False, YTReplacementDomain)
+                        False, YTReplacementDomain, newswire)
 
 
 def htmlInbox(defaultTimeline: str,
@@ -6084,7 +6100,8 @@ def htmlInbox(defaultTimeline: str,
               nickname: str, domain: str, port: int, inboxJson: {},
               allowDeletion: bool,
               httpPrefix: str, projectVersion: str,
-              minimal: bool, YTReplacementDomain: str) -> str:
+              minimal: bool, YTReplacementDomain: str,
+              newswire: {}) -> str:
     """Show the inbox as html
     """
     manuallyApproveFollowers = \
@@ -6096,7 +6113,7 @@ def htmlInbox(defaultTimeline: str,
                         nickname, domain, port, inboxJson,
                         'inbox', allowDeletion,
                         httpPrefix, projectVersion, manuallyApproveFollowers,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlBookmarks(defaultTimeline: str,
@@ -6106,7 +6123,8 @@ def htmlBookmarks(defaultTimeline: str,
                   nickname: str, domain: str, port: int, bookmarksJson: {},
                   allowDeletion: bool,
                   httpPrefix: str, projectVersion: str,
-                  minimal: bool, YTReplacementDomain: str) -> str:
+                  minimal: bool, YTReplacementDomain: str,
+                  newswire: {}) -> str:
     """Show the bookmarks as html
     """
     manuallyApproveFollowers = \
@@ -6118,7 +6136,7 @@ def htmlBookmarks(defaultTimeline: str,
                         nickname, domain, port, bookmarksJson,
                         'tlbookmarks', allowDeletion,
                         httpPrefix, projectVersion, manuallyApproveFollowers,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlEvents(defaultTimeline: str,
@@ -6128,7 +6146,8 @@ def htmlEvents(defaultTimeline: str,
                nickname: str, domain: str, port: int, bookmarksJson: {},
                allowDeletion: bool,
                httpPrefix: str, projectVersion: str,
-               minimal: bool, YTReplacementDomain: str) -> str:
+               minimal: bool, YTReplacementDomain: str,
+               newswire: {}) -> str:
     """Show the events as html
     """
     manuallyApproveFollowers = \
@@ -6140,7 +6159,7 @@ def htmlEvents(defaultTimeline: str,
                         nickname, domain, port, bookmarksJson,
                         'tlevents', allowDeletion,
                         httpPrefix, projectVersion, manuallyApproveFollowers,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlInboxDMs(defaultTimeline: str,
@@ -6150,7 +6169,8 @@ def htmlInboxDMs(defaultTimeline: str,
                  nickname: str, domain: str, port: int, inboxJson: {},
                  allowDeletion: bool,
                  httpPrefix: str, projectVersion: str,
-                 minimal: bool, YTReplacementDomain: str) -> str:
+                 minimal: bool, YTReplacementDomain: str,
+                 newswire: {}) -> str:
     """Show the DM timeline as html
     """
     return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts,
@@ -6158,7 +6178,7 @@ def htmlInboxDMs(defaultTimeline: str,
                         itemsPerPage, session, baseDir, wfRequest, personCache,
                         nickname, domain, port, inboxJson, 'dm', allowDeletion,
                         httpPrefix, projectVersion, False, minimal,
-                        YTReplacementDomain)
+                        YTReplacementDomain, newswire)
 
 
 def htmlInboxReplies(defaultTimeline: str,
@@ -6168,7 +6188,8 @@ def htmlInboxReplies(defaultTimeline: str,
                      nickname: str, domain: str, port: int, inboxJson: {},
                      allowDeletion: bool,
                      httpPrefix: str, projectVersion: str,
-                     minimal: bool, YTReplacementDomain: str) -> str:
+                     minimal: bool, YTReplacementDomain: str,
+                     newswire: {}) -> str:
     """Show the replies timeline as html
     """
     return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts,
@@ -6176,7 +6197,7 @@ def htmlInboxReplies(defaultTimeline: str,
                         itemsPerPage, session, baseDir, wfRequest, personCache,
                         nickname, domain, port, inboxJson, 'tlreplies',
                         allowDeletion, httpPrefix, projectVersion, False,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlInboxMedia(defaultTimeline: str,
@@ -6186,7 +6207,8 @@ def htmlInboxMedia(defaultTimeline: str,
                    nickname: str, domain: str, port: int, inboxJson: {},
                    allowDeletion: bool,
                    httpPrefix: str, projectVersion: str,
-                   minimal: bool, YTReplacementDomain: str) -> str:
+                   minimal: bool, YTReplacementDomain: str,
+                   newswire: {}) -> str:
     """Show the media timeline as html
     """
     return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts,
@@ -6194,7 +6216,7 @@ def htmlInboxMedia(defaultTimeline: str,
                         itemsPerPage, session, baseDir, wfRequest, personCache,
                         nickname, domain, port, inboxJson, 'tlmedia',
                         allowDeletion, httpPrefix, projectVersion, False,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlInboxBlogs(defaultTimeline: str,
@@ -6204,7 +6226,8 @@ def htmlInboxBlogs(defaultTimeline: str,
                    nickname: str, domain: str, port: int, inboxJson: {},
                    allowDeletion: bool,
                    httpPrefix: str, projectVersion: str,
-                   minimal: bool, YTReplacementDomain: str) -> str:
+                   minimal: bool, YTReplacementDomain: str,
+                   newswire: {}) -> str:
     """Show the blogs timeline as html
     """
     return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts,
@@ -6212,7 +6235,7 @@ def htmlInboxBlogs(defaultTimeline: str,
                         itemsPerPage, session, baseDir, wfRequest, personCache,
                         nickname, domain, port, inboxJson, 'tlblogs',
                         allowDeletion, httpPrefix, projectVersion, False,
-                        minimal, YTReplacementDomain)
+                        minimal, YTReplacementDomain, newswire)
 
 
 def htmlModeration(defaultTimeline: str,
@@ -6222,7 +6245,8 @@ def htmlModeration(defaultTimeline: str,
                    nickname: str, domain: str, port: int, inboxJson: {},
                    allowDeletion: bool,
                    httpPrefix: str, projectVersion: str,
-                   YTReplacementDomain: str) -> str:
+                   YTReplacementDomain: str,
+                   newswire: {}) -> str:
     """Show the moderation feed as html
     """
     return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts,
@@ -6230,7 +6254,7 @@ def htmlModeration(defaultTimeline: str,
                         itemsPerPage, session, baseDir, wfRequest, personCache,
                         nickname, domain, port, inboxJson, 'moderation',
                         allowDeletion, httpPrefix, projectVersion, True, False,
-                        YTReplacementDomain)
+                        YTReplacementDomain, newswire)
 
 
 def htmlOutbox(defaultTimeline: str,
@@ -6240,7 +6264,8 @@ def htmlOutbox(defaultTimeline: str,
                nickname: str, domain: str, port: int, outboxJson: {},
                allowDeletion: bool,
                httpPrefix: str, projectVersion: str,
-               minimal: bool, YTReplacementDomain: str) -> str:
+               minimal: bool, YTReplacementDomain: str,
+               newswire: {}) -> str:
     """Show the Outbox as html
     """
     manuallyApproveFollowers = \
@@ -6251,7 +6276,7 @@ def htmlOutbox(defaultTimeline: str,
                         nickname, domain, port, outboxJson, 'outbox',
                         allowDeletion, httpPrefix, projectVersion,
                         manuallyApproveFollowers, minimal,
-                        YTReplacementDomain)
+                        YTReplacementDomain, newswire)
 
 
 def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int,