Apply cw from lists

blocking.py

@@ -876,26 +876,81 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool:
     return False
 
 
-def loadLists(baseDir: str, verbose: bool) -> {}:
+def loadCWLists(baseDir: str, verbose: bool) -> {}:
-    """Load lists used for blocking or warnings
+    """Load lists used for content warnings
     """
-    if not os.path.isdir(baseDir + '/lists'):
+    if not os.path.isdir(baseDir + '/cwlists'):
         return {}
     result = {}
-    for subdir, dirs, files in os.walk(baseDir + '/lists'):
+    for subdir, dirs, files in os.walk(baseDir + '/cwlists'):
         for f in files:
             if not f.endswith('.json'):
                 continue
-            listFilename = os.path.join(baseDir + '/lists', f)
+            listFilename = os.path.join(baseDir + '/cwlists', f)
-            listJson = loadJson(listFilename)
+            print('listFilename: ' + listFilename)
+            listJson = loadJson(listFilename, 0, 1)
             if not listJson:
                 continue
             if not listJson.get('name'):
                 continue
-            if not listJson('words') and not listJson.get('domains'):
+            if not listJson.get('words') and not listJson.get('domains'):
                 continue
             name = listJson['name']
             if verbose:
                 print('List: ' + name)
             result[name] = listJson
     return result
+
+
+def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}) -> None:
+    """Adds content warnings by matching the post content
+    against domains or keywords
+    """
+    if not postJsonObject['object'].get('content'):
+        return
+    cw = ''
+    if postJsonObject['object'].get('summary'):
+        cw = postJsonObject['object']['summary']
+
+    content = postJsonObject['object']['content']
+    for name, item in CWlists.items():
+        if not item.get('warning'):
+            continue
+        warning = item['warning']
+
+        # is there a translated version of the warning?
+        if translate.get(warning):
+            warning = translate[warning]
+
+        # is the warning already in the CW?
+        if warning in cw:
+            continue
+
+        matched = False
+
+        # match domains within the content
+        if item.get('domains'):
+            for domain in item['domains']:
+                if domain in content:
+                    if cw:
+                        cw = warning + ' / ' + cw
+                    else:
+                        cw = warning
+                    matched = True
+                    break
+
+        if matched:
+            continue
+
+        # match words within the content
+        if item.get('words'):
+            for wordStr in item['words']:
+                if wordStr in content:
+                    if cw:
+                        cw = warning + ' / ' + cw
+                    else:
+                        cw = warning
+                    break
+    if cw:
+        postJsonObject['object']['summary'] = cw
+        postJsonObject['object']['sensitive'] = True

cwlists/murdoch.json

@@ -0,0 +1,171 @@
+{
+    "name": "Murdoch press",
+    "warning": "Murdoch Press",
+    "words": [],
+    "domains": [
+	"api.news",
+	"content.api.news",
+	"newscdn.com.au",
+	"resourcesssl.newscdn.com.au",
+	"thesun.co.uk",
+	"ads.thesun.co.uk",
+	"cdn.thesun.co.uk",
+	"cmp.cdn.thesun.co.uk",
+	"www.thesun.co.uk",
+	"thetimes.co.uk",
+	"ads.thetimes.co.uk",
+	"www.thetimes.co.uk",
+	"thesundaytimes.co.uk",
+	"pressassociation.com",
+	"news.co.uk",
+	"www.news.co.uk",
+	"newscorpaustralia.com",
+	"www.newscorpaustralia.com",
+	"logflume.newscorpaustralia.com",
+	"theaustralian.com.au",
+	"media.theaustralian.com.au",
+	"www.theaustralian.com.au",
+	"aap.com.au",
+	"www.aap.com.au",
+	"news.com.au",
+	"tags.news.com.au",
+	"go.news.com.au",
+	"origin.go.news.com.au",
+	"www.news.com.au",
+	"skynews.com.au",
+	"www.skynews.com.au",
+	"www.skyweather.com.au",
+	"skyweather.com.au",
+	"australiannewschannel.com.au",
+	"weeklytimesnow.com.au",
+	"www.weeklytimesnow.com.au",
+	"dailytelegraph.com.au",
+	"www.dailytelegraph.com.au",
+	"heraldsun.com.au",
+	"www.heraldsun.com.au",
+	"geelongadvertiser.com.au",
+	"www.geelongadvertiser.com.au",
+	"couriermail.com.au",
+	"www.couriermail.com.au",
+	"thesundaymail.com.au",
+	"goldcoastbulletin.com.au",
+	"www.goldcoastbulletin.com.au",
+	"cairnspost.com.au",
+	"www.cairnspost.com.au",
+	"townsvillebulletin.com.au",
+	"www.townsvillebulletin.com.au",
+	"adelaidenow.com.au",
+	"www.adelaidenow.com.au",
+	"themercury.com.au",
+	"www.themercury.com.au",
+	"ntnews.com.au",
+	"www.ntnews.com.au",
+	"postcourier.com.pg",
+	"nypost.com",
+	"embeds.nypost.com",
+	"pagesix.com",
+	"realtor.com",
+	"wsj.com",
+	"www.wsj.com",
+	"accounts.wsj.com",
+	"video-api.wsj.com",
+	"wsj.net",
+	"images.wsj.net",
+	"m4.wsj.net",
+	"m.wsj.net",
+	"s.wsj.net",
+	"si.wsj.net",
+	"sts3.wsj.net",
+	"vir.wsj.net",
+	"foxnews.com",
+	"www.foxnews.com",
+	"a57.foxnews.com",
+	"static.foxnews.com",
+	"fncstatic.com",
+	"global.fncstatic.com",
+	"foxnewsgo.com",
+	"www.foxbusiness.com",
+	"fox.com",
+	"strike.fox.com",
+	"foxbusiness.com",
+	"www.foxbusiness.com",
+	"foxsports.com",
+	"fssta.com",
+	"b.fssta.com",
+	"foxsports.com.au",
+	"dowjones.com",
+	"www.dowjones.com",
+	"images.dowjones.com",
+	"factiva.com",
+	"barrons.com",
+	"accounts.barrons.com",
+	"asset.barrons.com",
+	"images.barrons.com",
+	"video-api.barrons.com",
+	"www.barrons.com",
+	"marketwatch.com",
+	"www.marketwatch.com",
+	"heatst.com",
+	"fnlondon.com",
+	"www.fnlondon.com",
+	"assets.fnlondon.com",
+	"images.fnlondon.com",
+	"mansionglobal.com",
+	"images.mansionglobal.com",
+	"www.mansionglobal.com",
+	"spindices.com",
+	"spglobal.com",
+	"www.spglobal.com",
+	"talksport.com",
+	"ads.talksport.com",
+	"cdn259.talksport.com",
+	"harpercollins.com",
+	"www.harpercollins.com",
+	"bestrecipes.com.au",
+	"img.bestrecipes.com.au",
+	"www.bestrecipes.com.au",
+	"hipages.com.au",
+	"img.hipages.com.au",
+	"homeimprovementpages.com.au",
+	"assets.homeimprovementpages.com.au",
+	"odds.com.au",
+	"www.odds.com.au",
+	"onebigswitch.com.au",
+	"suddenly.com.au",
+	"www.suddenly.com.au",
+	"supercoach.com.au",
+	"punters.com.au",
+	"www.punters.com.au",
+	"kayosports.com.au",
+	"foxtel.com.au",
+	"www.foxtel.com.au",
+	"newscorp.com",
+	"storyful.com",
+	"vogue.com.au",
+	"media.vogue.com.au",
+	"www.vogue.com.au",
+	"taste.com.au",
+	"img.taste.com.au",
+	"www.taste.com.au",
+	"kidspot.com.au",
+	"go.kidspot.com.au",
+	"origin.go.kidspot.com.au",
+	"img.kidspot.com.au",
+	"www.kidspot.com.au",
+	"bodyandsoul.com.au",
+	"go.bodyandsoul.com.au",
+	"origin.bodyandsoul.com.au",
+	"www.bodyandsoul.com.au",
+	"realcommercial.com.au",
+	"www.realcommercial.com.au",
+	"reastatic.net",
+	"au.reastatic.net",
+	"rui.au.reastatic.net",
+	"s1.rui.au.reastatic.net",
+	"realestate.com.au",
+	"www.realestate.com.au",
+	"whereilive.com.au",
+	"www.newsoftheworld.co.uk",
+	"newsoftheworld.co.uk"
+    ]
+}

daemon.py

@@ -125,7 +125,7 @@ from media import replaceTwitter
 from media import attachMedia
 from media import pathIsVideo
 from media import pathIsAudio
-from blocking import loadLists
+from blocking import loadCWLists
 from blocking import updateBlockedCache
 from blocking import mutePost
 from blocking import unmutePost
@@ -1230,7 +1230,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.peertubeInstances,
                                    self.server.themeName,
                                    self.server.maxLikeCount,
-                                   self.server.maxRecentPosts)
+                                   self.server.maxRecentPosts,
+                                   self.server.CWlists)
 
     def _postToOutboxThread(self, messageJson: {}) -> bool:
         """Creates a thread to send a post
@@ -3008,7 +3009,8 @@ class PubServer(BaseHTTPRequestHandler):
                                       self.server.themeName,
                                       self.server.systemLanguage,
                                       self.server.maxLikeCount,
-                                      self.server.signingPrivateKeyPem)
+                                      self.server.signingPrivateKeyPem,
+                                      self.server.CWlists)
                 if hashtagStr:
                     msg = hashtagStr.encode('utf-8')
                     msglen = len(msg)
@@ -3066,7 +3068,8 @@ class PubServer(BaseHTTPRequestHandler):
                                       self.server.themeName, 'outbox',
                                       self.server.systemLanguage,
                                       self.server.maxLikeCount,
-                                      self.server.signingPrivateKeyPem)
+                                      self.server.signingPrivateKeyPem,
+                                      self.server.CWlists)
                 if historyStr:
                     msg = historyStr.encode('utf-8')
                     msglen = len(msg)
@@ -3104,7 +3107,8 @@ class PubServer(BaseHTTPRequestHandler):
                                       self.server.themeName, 'bookmarks',
                                       self.server.systemLanguage,
                                       self.server.maxLikeCount,
-                                      self.server.signingPrivateKeyPem)
+                                      self.server.signingPrivateKeyPem,
+                                      self.server.CWlists)
                 if bookmarksStr:
                     msg = bookmarksStr.encode('utf-8')
                     msglen = len(msg)
@@ -3208,7 +3212,8 @@ class PubServer(BaseHTTPRequestHandler):
                                                accessKeys,
                                                self.server.systemLanguage,
                                                self.server.maxLikeCount,
-                                               signingPrivateKeyPem)
+                                               signingPrivateKeyPem,
+                                               self.server.CWlists)
                 if profileStr:
                     msg = profileStr.encode('utf-8')
                     msglen = len(msg)
@@ -6728,7 +6733,8 @@ class PubServer(BaseHTTPRequestHandler):
                               self.server.themeName,
                               self.server.systemLanguage,
                               self.server.maxLikeCount,
-                              self.server.signingPrivateKeyPem)
+                              self.server.signingPrivateKeyPem,
+                              self.server.CWlists)
         if hashtagStr:
             msg = hashtagStr.encode('utf-8')
             msglen = len(msg)
@@ -6950,7 +6956,8 @@ class PubServer(BaseHTTPRequestHandler):
                                  showRepeats,
                                  showIndividualPostIcons,
                                  manuallyApproveFollowers,
-                                 False, True, False)
+                                 False, True, False,
+                                 self.server.CWlists)
 
         self.server.GETbusy = False
         actorAbsolute = self._getInstanceUrl(callingDomain) + actor
@@ -7431,7 +7438,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showRepeats,
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
-                                     False, True, False)
+                                     False, True, False,
+                                     self.server.CWlists)
             else:
                 print('WARN: Liked post not found: ' + likedPostFilename)
             # clear the icon from the cache so that it gets updated
@@ -7591,7 +7599,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showRepeats,
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
-                                     False, True, False)
+                                     False, True, False,
+                                     self.server.CWlists)
             else:
                 print('WARN: Unliked post not found: ' + likedPostFilename)
             # clear the icon from the cache so that it gets updated
@@ -7726,7 +7735,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showRepeats,
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
-                                     False, True, False)
+                                     False, True, False,
+                                     self.server.CWlists)
             else:
                 print('WARN: Bookmarked post not found: ' + bookmarkFilename)
         # self._postToOutbox(bookmarkJson, self.server.projectVersion, None)
@@ -7860,7 +7870,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showRepeats,
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
-                                     False, True, False)
+                                     False, True, False,
+                                     self.server.CWlists)
             else:
                 print('WARN: Unbookmarked post not found: ' + bookmarkFilename)
         self.server.GETbusy = False
@@ -7962,7 +7973,8 @@ class PubServer(BaseHTTPRequestHandler):
                                   self.server.themeName,
                                   self.server.systemLanguage,
                                   self.server.maxLikeCount,
-                                  self.server.signingPrivateKeyPem)
+                                  self.server.signingPrivateKeyPem,
+                                  self.server.CWlists)
             if deleteStr:
                 deleteStrLen = len(deleteStr)
                 self._set_headers('text/html', deleteStrLen,
@@ -8070,7 +8082,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
                                      showPublicOnly, storeToCache,
-                                     useCacheOnly)
+                                     useCacheOnly,
+                                     self.server.CWlists)
             else:
                 print('WARN: Muted post not found: ' + muteFilename)
 
@@ -8179,7 +8192,8 @@ class PubServer(BaseHTTPRequestHandler):
                                      showIndividualPostIcons,
                                      manuallyApproveFollowers,
                                      showPublicOnly, storeToCache,
-                                     useCacheOnly)
+                                     useCacheOnly,
+                                     self.server.CWlists)
             else:
                 print('WARN: Unmuted post not found: ' + muteFilename)
         self.server.GETbusy = False
@@ -8302,7 +8316,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.themeName,
                                     self.server.systemLanguage,
                                     self.server.maxLikeCount,
-                                    self.server.signingPrivateKeyPem)
+                                    self.server.signingPrivateKeyPem,
+                                    self.server.CWlists)
                 msg = msg.encode('utf-8')
                 msglen = len(msg)
                 self._set_headers('text/html', msglen,
@@ -8401,7 +8416,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.themeName,
                                     self.server.systemLanguage,
                                     self.server.maxLikeCount,
-                                    self.server.signingPrivateKeyPem)
+                                    self.server.signingPrivateKeyPem,
+                                    self.server.CWlists)
                 msg = msg.encode('utf-8')
                 msglen = len(msg)
                 self._set_headers('text/html', msglen,
@@ -8510,7 +8526,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.maxLikeCount,
                                     self.server.sharedItemsFederatedDomains,
                                     rolesList,
-                                    None, None)
+                                    None, None, self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -8623,7 +8639,8 @@ class PubServer(BaseHTTPRequestHandler):
                                                 self.server.maxLikeCount,
                                                 sharedItemsFederatedDomains,
                                                 skills,
-                                                None, None)
+                                                None, None,
+                                                self.server.CWlists)
                                 msg = msg.encode('utf-8')
                                 msglen = len(msg)
                                 self._set_headers('text/html', msglen,
@@ -8772,7 +8789,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.themeName,
                                    self.server.systemLanguage,
                                    self.server.maxLikeCount,
-                                   self.server.signingPrivateKeyPem)
+                                   self.server.signingPrivateKeyPem,
+                                   self.server.CWlists)
             msg = msg.encode('utf-8')
             msglen = len(msg)
             self._set_headers('text/html', msglen,
@@ -9018,7 +9036,8 @@ class PubServer(BaseHTTPRequestHandler):
                                         self.server.systemLanguage,
                                         self.server.maxLikeCount,
                                         sharedItemsFederatedDomains,
-                                        self.server.signingPrivateKeyPem)
+                                        self.server.signingPrivateKeyPem,
+                                        self.server.CWlists)
                         if GETstartTime:
                             fitnessPerformance(GETstartTime,
                                                self.server.fitness,
@@ -9166,7 +9185,8 @@ class PubServer(BaseHTTPRequestHandler):
                                          self.server.systemLanguage,
                                          self.server.maxLikeCount,
                                          sharedItemsFederatedDomains,
-                                         self.server.signingPrivateKeyPem)
+                                         self.server.signingPrivateKeyPem,
+                                         self.server.CWlists)
                         msg = msg.encode('utf-8')
                         msglen = len(msg)
                         self._set_headers('text/html', msglen,
@@ -9307,7 +9327,8 @@ class PubServer(BaseHTTPRequestHandler):
                                          self.server.systemLanguage,
                                          self.server.maxLikeCount,
                                          sharedItemsFederatedDomains,
-                                         self.server.signingPrivateKeyPem)
+                                         self.server.signingPrivateKeyPem,
+                                         self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -9447,7 +9468,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        self.server.systemLanguage,
                                        self.server.maxLikeCount,
                                        self.server.sharedItemsFederatedDomains,
-                                       self.server.signingPrivateKeyPem)
+                                       self.server.signingPrivateKeyPem,
+                                       self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -9587,7 +9609,8 @@ class PubServer(BaseHTTPRequestHandler):
                                        self.server.systemLanguage,
                                        self.server.maxLikeCount,
                                        self.server.sharedItemsFederatedDomains,
-                                       self.server.signingPrivateKeyPem)
+                                       self.server.signingPrivateKeyPem,
+                                       self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -9736,7 +9759,8 @@ class PubServer(BaseHTTPRequestHandler):
                                       self.server.systemLanguage,
                                       self.server.maxLikeCount,
                                       self.server.sharedItemsFederatedDomains,
-                                      self.server.signingPrivateKeyPem)
+                                      self.server.signingPrivateKeyPem,
+                                      self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -9883,7 +9907,8 @@ class PubServer(BaseHTTPRequestHandler):
                                           self.server.systemLanguage,
                                           self.server.maxLikeCount,
                                           sharedItemsFederatedDomains,
-                                          self.server.signingPrivateKeyPem)
+                                          self.server.signingPrivateKeyPem,
+                                          self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -9989,7 +10014,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.systemLanguage,
                                    self.server.maxLikeCount,
                                    self.server.sharedItemsFederatedDomains,
-                                   self.server.signingPrivateKeyPem)
+                                   self.server.signingPrivateKeyPem,
+                                   self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -10073,7 +10099,8 @@ class PubServer(BaseHTTPRequestHandler):
                                    self.server.systemLanguage,
                                    self.server.maxLikeCount,
                                    self.server.sharedItemsFederatedDomains,
-                                   self.server.signingPrivateKeyPem)
+                                   self.server.signingPrivateKeyPem,
+                                   self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -10194,7 +10221,8 @@ class PubServer(BaseHTTPRequestHandler):
                                           self.server.systemLanguage,
                                           self.server.maxLikeCount,
                                           sharedItemsFederatedDomains,
-                                          self.server.signingPrivateKeyPem)
+                                          self.server.signingPrivateKeyPem,
+                                          self.server.CWlists)
                         msg = msg.encode('utf-8')
                         msglen = len(msg)
                         self._set_headers('text/html', msglen,
@@ -10332,7 +10360,8 @@ class PubServer(BaseHTTPRequestHandler):
                                self.server.systemLanguage,
                                self.server.maxLikeCount,
                                self.server.sharedItemsFederatedDomains,
-                               self.server.signingPrivateKeyPem)
+                               self.server.signingPrivateKeyPem,
+                               self.server.CWlists)
                 msg = msg.encode('utf-8')
                 msglen = len(msg)
                 self._set_headers('text/html', msglen,
@@ -10461,7 +10490,8 @@ class PubServer(BaseHTTPRequestHandler):
                                            self.server.systemLanguage,
                                            self.server.maxLikeCount,
                                            sharedItemsFederatedDomains,
-                                           self.server.signingPrivateKeyPem)
+                                           self.server.signingPrivateKeyPem,
+                                           self.server.CWlists)
                         msg = msg.encode('utf-8')
                         msglen = len(msg)
                         self._set_headers('text/html', msglen,
@@ -10588,7 +10618,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.maxLikeCount,
                                     self.server.sharedItemsFederatedDomains,
                                     shares,
-                                    pageNumber, sharesPerPage)
+                                    pageNumber, sharesPerPage,
+                                    self.server.CWlists)
                     msg = msg.encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
@@ -10712,7 +10743,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.sharedItemsFederatedDomains,
                                     following,
                                     pageNumber,
-                                    followsPerPage).encode('utf-8')
+                                    followsPerPage,
+                                    self.server.CWlists).encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html',
                                       msglen, cookie, callingDomain, False)
@@ -10835,7 +10867,8 @@ class PubServer(BaseHTTPRequestHandler):
                                     self.server.sharedItemsFederatedDomains,
                                     followers,
                                     pageNumber,
-                                    followsPerPage).encode('utf-8')
+                                    followsPerPage,
+                                    self.server.CWlists).encode('utf-8')
                     msglen = len(msg)
                     self._set_headers('text/html', msglen,
                                       cookie, callingDomain, False)
@@ -10974,7 +11007,8 @@ class PubServer(BaseHTTPRequestHandler):
                             self.server.systemLanguage,
                             self.server.maxLikeCount,
                             self.server.sharedItemsFederatedDomains,
-                            None, None).encode('utf-8')
+                            None, None, None,
+                            self.server.CWlists).encode('utf-8')
             msglen = len(msg)
             self._set_headers('text/html', msglen,
                               cookie, callingDomain, False)
@@ -17165,7 +17199,7 @@ def runDaemon(defaultReplyIntervalHours: int,
         updateBlockedCache(baseDir, httpd.blockedCache,
                            httpd.blockedCacheLastUpdated,
                            httpd.blockedCacheUpdateSecs)
-    httpd.lists = loadLists(baseDir, True)
+    httpd.CWlists = loadCWLists(baseDir, True)
 
     # cache to store css files
     httpd.cssCache = {}
@@ -17315,7 +17349,8 @@ def runDaemon(defaultReplyIntervalHours: int,
                               httpd.systemLanguage,
                               httpd.maxLikeCount,
                               httpd.signingPrivateKeyPem,
-                              httpd.defaultReplyIntervalHours), daemon=True)
+                              httpd.defaultReplyIntervalHours,
+                              httpd.CWlists), daemon=True)
 
     print('Creating scheduled post thread')
     httpd.thrPostSchedule = \

inbox.py

@@ -274,7 +274,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
                                allowLocalNetworkAccess: bool,
                                themeName: str, systemLanguage: str,
                                maxLikeCount: int,
-                               signingPrivateKeyPem: str) -> None:
+                               signingPrivateKeyPem: str,
+                               CWlists: {}) -> None:
     """Converts the json post into html and stores it in a cache
     This enables the post to be quickly displayed later
     """
@@ -298,7 +299,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
                          showPublishedDateOnly,
                          peertubeInstances, allowLocalNetworkAccess,
                          themeName, systemLanguage, maxLikeCount,
-                         notDM, True, True, False, True, False)
+                         notDM, True, True, False, True, False,
+                         CWlists)
 
 
 def validInbox(baseDir: str, nickname: str, domain: str) -> bool:
@@ -980,7 +982,7 @@ def _receiveLike(recentPostsCache: {},
                  peertubeInstances: [],
                  allowLocalNetworkAccess: bool,
                  themeName: str, systemLanguage: str,
-                 maxLikeCount: int) -> bool:
+                 maxLikeCount: int, CWlists: {}) -> bool:
     """Receives a Like activity within the POST section of HTTPServer
     """
     if messageJson['type'] != 'Like':
@@ -1082,7 +1084,7 @@ def _receiveLike(recentPostsCache: {},
                                  maxLikeCount, notDM,
                                  showIndividualPostIcons,
                                  manuallyApproveFollowers,
-                                 False, True, False)
+                                 False, True, False, CWlists)
     return True
 
 
@@ -1100,7 +1102,7 @@ def _receiveUndoLike(recentPostsCache: {},
                      peertubeInstances: [],
                      allowLocalNetworkAccess: bool,
                      themeName: str, systemLanguage: str,
-                     maxLikeCount: int) -> bool:
+                     maxLikeCount: int, CWlists: {}) -> bool:
     """Receives an undo like activity within the POST section of HTTPServer
     """
     if messageJson['type'] != 'Undo':
@@ -1191,7 +1193,7 @@ def _receiveUndoLike(recentPostsCache: {},
                                  maxLikeCount, notDM,
                                  showIndividualPostIcons,
                                  manuallyApproveFollowers,
-                                 False, True, False)
+                                 False, True, False, CWlists)
     return True
 
 
@@ -1208,7 +1210,7 @@ def _receiveBookmark(recentPostsCache: {},
                      peertubeInstances: [],
                      allowLocalNetworkAccess: bool,
                      themeName: str, systemLanguage: str,
-                     maxLikeCount: int) -> bool:
+                     maxLikeCount: int, CWlists: {}) -> bool:
     """Receives a bookmark activity within the POST section of HTTPServer
     """
     if not messageJson.get('type'):
@@ -1299,7 +1301,7 @@ def _receiveBookmark(recentPostsCache: {},
                              maxLikeCount, notDM,
                              showIndividualPostIcons,
                              manuallyApproveFollowers,
-                             False, True, False)
+                             False, True, False, CWlists)
     return True
 
 
@@ -1316,7 +1318,7 @@ def _receiveUndoBookmark(recentPostsCache: {},
                          peertubeInstances: [],
                          allowLocalNetworkAccess: bool,
                          themeName: str, systemLanguage: str,
-                         maxLikeCount: int) -> bool:
+                         maxLikeCount: int, CWlists: {}) -> bool:
     """Receives an undo bookmark activity within the POST section of HTTPServer
     """
     if not messageJson.get('type'):
@@ -1408,7 +1410,7 @@ def _receiveUndoBookmark(recentPostsCache: {},
                              maxLikeCount, notDM,
                              showIndividualPostIcons,
                              manuallyApproveFollowers,
-                             False, True, False)
+                             False, True, False, CWlists)
     return True
 
 
@@ -1502,7 +1504,7 @@ def _receiveAnnounce(recentPostsCache: {},
                      maxRecentPosts: int,
                      allowDeletion: bool,
                      peertubeInstances: [],
-                     maxLikeCount: int) -> bool:
+                     maxLikeCount: int, CWlists: {}) -> bool:
     """Receives an announce activity within the POST section of HTTPServer
     """
     if messageJson['type'] != 'Announce':
@@ -1614,7 +1616,7 @@ def _receiveAnnounce(recentPostsCache: {},
                              maxLikeCount, notDM,
                              showIndividualPostIcons,
                              manuallyApproveFollowers,
-                             False, True, False)
+                             False, True, False, CWlists)
     if not announceHtml:
         print('WARN: Unable to generate html for announce ' +
               str(messageJson))
@@ -2551,7 +2553,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                        themeName: str, systemLanguage: str,
                        maxLikeCount: int,
                        signingPrivateKeyPem: str,
-                       defaultReplyIntervalHours: int) -> bool:
+                       defaultReplyIntervalHours: int,
+                       CWlists: {}) -> bool:
     """ Anything which needs to be done after initial checks have passed
     """
     actor = keyId
@@ -2581,7 +2584,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                     peertubeInstances,
                     allowLocalNetworkAccess,
                     themeName, systemLanguage,
-                    maxLikeCount):
+                    maxLikeCount, CWlists):
         if debug:
             print('DEBUG: Like accepted from ' + actor)
         return False
@@ -2603,7 +2606,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                         peertubeInstances,
                         allowLocalNetworkAccess,
                         themeName, systemLanguage,
-                        maxLikeCount):
+                        maxLikeCount, CWlists):
         if debug:
             print('DEBUG: Undo like accepted from ' + actor)
         return False
@@ -2625,7 +2628,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                         peertubeInstances,
                         allowLocalNetworkAccess,
                         themeName, systemLanguage,
-                        maxLikeCount):
+                        maxLikeCount, CWlists):
         if debug:
             print('DEBUG: Bookmark accepted from ' + actor)
         return False
@@ -2647,7 +2650,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                             peertubeInstances,
                             allowLocalNetworkAccess,
                             themeName, systemLanguage,
-                            maxLikeCount):
+                            maxLikeCount, CWlists):
         if debug:
             print('DEBUG: Undo bookmark accepted from ' + actor)
         return False
@@ -2673,7 +2676,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                         maxRecentPosts,
                         allowDeletion,
                         peertubeInstances,
-                        maxLikeCount):
+                        maxLikeCount, CWlists):
         if debug:
             print('DEBUG: Announce accepted from ' + actor)
 
@@ -2954,7 +2957,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                                                    allowLocalNetworkAccess,
                                                    themeName, systemLanguage,
                                                    maxLikeCount,
-                                                   signingPrivateKeyPem)
+                                                   signingPrivateKeyPem,
+                                                   CWlists)
                         if debug:
                             timeDiff = \
                                 str(int((time.time() - htmlCacheStartTime) *
@@ -3239,7 +3243,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
                   verifyAllSignatures: bool,
                   themeName: str, systemLanguage: str,
                   maxLikeCount: int, signingPrivateKeyPem: str,
-                  defaultReplyIntervalHours: int) -> None:
+                  defaultReplyIntervalHours: int,
+                  CWlists: {}) -> None:
     """Processes received items and moves them to the appropriate
     directories
     """
@@ -3656,7 +3661,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
                                themeName, systemLanguage,
                                maxLikeCount,
                                signingPrivateKeyPem,
-                               defaultReplyIntervalHours)
+                               defaultReplyIntervalHours,
+                               CWlists)
             if debug:
                 pprint(queueJson['post'])
                 print('Queue: Queue post accepted')

lists/murdoch.json

@@ -1,171 +0,0 @@
-{
-    "name": "Murdoch press",
-    "warning": "Murdoch Press",
-    "words": [],
-    "domains": [
-	api.news,
-	content.api.news,
-	newscdn.com.au,
-	resourcesssl.newscdn.com.au,
-	thesun.co.uk,
-	ads.thesun.co.uk,
-	cdn.thesun.co.uk,
-	cmp.cdn.thesun.co.uk,
-	www.thesun.co.uk,
-	thetimes.co.uk,
-	ads.thetimes.co.uk,
-	www.thetimes.co.uk,
-	thesundaytimes.co.uk,
-	pressassociation.com,
-	news.co.uk,
-	www.news.co.uk,
-	newscorpaustralia.com,
-	www.newscorpaustralia.com,
-	logflume.newscorpaustralia.com,
-	theaustralian.com.au,
-	media.theaustralian.com.au,
-	www.theaustralian.com.au,
-	aap.com.au,
-	www.aap.com.au,
-	news.com.au,
-	tags.news.com.au,
-	go.news.com.au,
-	origin.go.news.com.au,
-	www.news.com.au,
-	skynews.com.au,
-	www.skynews.com.au,
-	www.skyweather.com.au,
-	skyweather.com.au,
-	australiannewschannel.com.au,
-	weeklytimesnow.com.au,
-	www.weeklytimesnow.com.au,
-	dailytelegraph.com.au,
-	www.dailytelegraph.com.au,
-	heraldsun.com.au,
-	www.heraldsun.com.au,
-	geelongadvertiser.com.au,
-	www.geelongadvertiser.com.au,
-	couriermail.com.au,
-	www.couriermail.com.au,
-	thesundaymail.com.au,
-	goldcoastbulletin.com.au,
-	www.goldcoastbulletin.com.au,
-	cairnspost.com.au,
-	www.cairnspost.com.au,
-	townsvillebulletin.com.au,
-	www.townsvillebulletin.com.au,
-	adelaidenow.com.au,
-	www.adelaidenow.com.au,
-	themercury.com.au,
-	www.themercury.com.au,
-	ntnews.com.au,
-	www.ntnews.com.au,
-	postcourier.com.pg,
-	nypost.com,
-	embeds.nypost.com,
-	pagesix.com,
-	realtor.com,
-	wsj.com,
-	www.wsj.com,
-	accounts.wsj.com,
-	video-api.wsj.com,
-	wsj.net,
-	images.wsj.net,
-	m4.wsj.net,
-	m.wsj.net,
-	s.wsj.net,
-	si.wsj.net,
-	sts3.wsj.net,
-	vir.wsj.net,
-	foxnews.com,
-	www.foxnews.com,
-	a57.foxnews.com,
-	static.foxnews.com,
-	fncstatic.com,
-	global.fncstatic.com,
-	foxnewsgo.com,
-	www.foxbusiness.com,
-	fox.com,
-	strike.fox.com,
-	foxbusiness.com,
-	www.foxbusiness.com,
-	foxsports.com,
-	fssta.com,
-	b.fssta.com,
-	foxsports.com.au,
-	dowjones.com,
-	www.dowjones.com,
-	images.dowjones.com,
-	factiva.com,
-	barrons.com,
-	accounts.barrons.com,
-	asset.barrons.com,
-	images.barrons.com,
-	video-api.barrons.com,
-	www.barrons.com,
-	marketwatch.com,
-	www.marketwatch.com,
-	heatst.com,
-	fnlondon.com,
-	www.fnlondon.com,
-	assets.fnlondon.com,
-	images.fnlondon.com,
-	mansionglobal.com,
-	images.mansionglobal.com,
-	www.mansionglobal.com,
-	spindices.com,
-	spglobal.com,
-	www.spglobal.com,
-	talksport.com,
-	ads.talksport.com,
-	cdn259.talksport.com,
-	harpercollins.com,
-	www.harpercollins.com,
-	bestrecipes.com.au,
-	img.bestrecipes.com.au,
-	www.bestrecipes.com.au,
-	hipages.com.au,
-	img.hipages.com.au,
-	homeimprovementpages.com.au,
-	assets.homeimprovementpages.com.au,
-	odds.com.au,
-	www.odds.com.au,
-	onebigswitch.com.au,
-	suddenly.com.au,
-	www.suddenly.com.au,
-	supercoach.com.au,
-	punters.com.au,
-	www.punters.com.au,
-	kayosports.com.au,
-	foxtel.com.au,
-	www.foxtel.com.au,
-	newscorp.com,
-	storyful.com,
-	vogue.com.au,
-	media.vogue.com.au,
-	www.vogue.com.au,
-	taste.com.au,
-	img.taste.com.au,
-	www.taste.com.au,
-	kidspot.com.au,
-	go.kidspot.com.au,
-	origin.go.kidspot.com.au,
-	img.kidspot.com.au,
-	www.kidspot.com.au,
-	bodyandsoul.com.au,
-	go.bodyandsoul.com.au,
-	origin.bodyandsoul.com.au,
-	www.bodyandsoul.com.au,
-	realcommercial.com.au,
-	www.realcommercial.com.au,
-	reastatic.net,
-	au.reastatic.net,
-	rui.au.reastatic.net,
-	s1.rui.au.reastatic.net,
-	realestate.com.au,
-	www.realestate.com.au,
-	whereilive.com.au,
-	www.newsoftheworld.co.uk,
-	newsoftheworld.co.uk
-    ]
-}

outbox.py

@@ -196,7 +196,7 @@ def postMessageToOutbox(session, translate: {},
                         signingPrivateKeyPem: str,
                         peertubeInstances: str, theme: str,
                         maxLikeCount: int,
-                        maxRecentPosts: int) -> bool:
+                        maxRecentPosts: int, CWlists: {}) -> bool:
     """post is received by the outbox
     Client to server message post
     https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
@@ -467,7 +467,8 @@ def postMessageToOutbox(session, translate: {},
                                          boxNameIndex != 'dm',
                                          showIndividualPostIcons,
                                          manuallyApproveFollowers,
-                                         False, True, useCacheOnly)
+                                         False, True, useCacheOnly,
+                                         CWlists)
 
     if outboxAnnounce(recentPostsCache,
                       baseDir, messageJson, debug):

schedule.py

@@ -124,7 +124,8 @@ def _updatePostSchedule(baseDir: str, handle: str, httpd,
                                        httpd.peertubeInstances,
                                        httpd.themeName,
                                        httpd.maxLikeCount,
-                                       httpd.maxRecentPosts):
+                                       httpd.maxRecentPosts,
+                                       httpd.CWlists):
                 indexLines.remove(line)
                 try:
                     os.remove(postFilename)

tests.py

@@ -164,6 +164,8 @@ from shares import updateSharedItemFederationToken
 from shares import mergeSharedItemTokens
 from shares import sendShareViaServer
 from shares import getSharedItemsCatalogViaServer
+from blocking import loadCWLists
+from blocking import addCWfromLists
 
 testServerGroupRunning = False
 testServerAliceRunning = False
@@ -5764,6 +5766,46 @@ def _testWordsSimilarity() -> None:
     assert similarity > 80
 
 
+def _testAddCWfromLists(baseDir: str) -> None:
+    print('testAddCWfromLists')
+    translate = {}
+    CWlists = loadCWLists(baseDir, True)
+    assert CWlists
+
+    postJsonObject = {
+        "object": {
+            "sensitive": False,
+            "summary": None,
+            "content": ""
+        }
+    }
+    addCWfromLists(postJsonObject, CWlists, translate)
+    assert postJsonObject['object']['sensitive'] is False
+    assert postJsonObject['object']['summary'] is None
+
+    postJsonObject = {
+        "object": {
+            "sensitive": False,
+            "summary": None,
+            "content": "Blah blah news.co.uk blah blah"
+        }
+    }
+    addCWfromLists(postJsonObject, CWlists, translate)
+    assert postJsonObject['object']['sensitive'] is True
+    assert postJsonObject['object']['summary'] == "Murdoch Press"
+
+    postJsonObject = {
+        "object": {
+            "sensitive": True,
+            "summary": "Existing CW",
+            "content": "Blah blah news.co.uk blah blah"
+        }
+    }
+    addCWfromLists(postJsonObject, CWlists, translate)
+    assert postJsonObject['object']['sensitive'] is True
+    assert postJsonObject['object']['summary'] == "Murdoch Press / Existing CW"
+
+
 def runAllTests():
     baseDir = os.getcwd()
     print('Running tests...')
@@ -5771,6 +5813,7 @@ def runAllTests():
     _translateOntology(baseDir)
     _testGetPriceFromString()
     _testFunctions()
+    _testAddCWfromLists(baseDir)
     _testWordsSimilarity()
     _testSecondsBetweenPublished()
     _testSignAndVerify()

webapp_confirm.py

@@ -35,7 +35,8 @@ def htmlConfirmDelete(cssCache: {},
                       peertubeInstances: [],
                       allowLocalNetworkAccess: bool,
                       themeName: str, systemLanguage: str,
-                      maxLikeCount: int, signingPrivateKeyPem: str) -> str:
+                      maxLikeCount: int, signingPrivateKeyPem: str,
+                      CWlists: {}) -> str:
     """Shows a screen asking to confirm the deletion of a post
     """
     if '/statuses/' not in messageId:
@@ -79,7 +80,8 @@ def htmlConfirmDelete(cssCache: {},
                              showPublishedDateOnly,
                              peertubeInstances, allowLocalNetworkAccess,
                              themeName, systemLanguage, maxLikeCount,
-                             False, False, False, False, False, False)
+                             False, False, False, False, False, False,
+                             CWlists)
     deletePostStr += '<center>'
     deletePostStr += \
         '  <p class="followText">' + \

webapp_frontscreen.py

@@ -35,7 +35,7 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int,
                           allowLocalNetworkAccess: bool,
                           themeName: str, systemLanguage: str,
                           maxLikeCount: int,
-                          signingPrivateKeyPem: str) -> str:
+                          signingPrivateKeyPem: str, CWlists: {}) -> str:
     """Shows posts on the front screen of a news instance
     These should only be public blog posts from the features timeline
     which is the blog timeline of the news actor
@@ -81,7 +81,8 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int,
                                          themeName, systemLanguage,
                                          maxLikeCount,
                                          False, False, False,
-                                         True, False, False)
+                                         True, False, False,
+                                         CWlists)
                 if postStr:
                     profileStr += postStr + separatorStr
                     ctr += 1
@@ -109,9 +110,10 @@ def htmlFrontScreen(signingPrivateKeyPem: str,
                     accessKeys: {},
                     systemLanguage: str, maxLikeCount: int,
                     sharedItemsFederatedDomains: [],
-                    extraJson: {} = None,
+                    extraJson: {},
-                    pageNumber: int = None,
+                    pageNumber: int,
-                    maxItemsPerPage: int = None) -> str:
+                    maxItemsPerPage: int,
+                    CWlists: {}) -> str:
     """Show the news instance front screen
     """
     nickname = profileJson['preferredUsername']
@@ -182,7 +184,8 @@ def htmlFrontScreen(signingPrivateKeyPem: str,
                               allowLocalNetworkAccess,
                               theme, systemLanguage,
                               maxLikeCount,
-                              signingPrivateKeyPem) + licenseStr
+                              signingPrivateKeyPem,
+                              CWlists) + licenseStr
 
     # Footer which is only used for system accounts
     profileFooterStr = '      </td>\n'

webapp_moderation.py

@@ -52,7 +52,8 @@ def htmlModeration(cssCache: {}, defaultTimeline: str,
                    accessKeys: {}, systemLanguage: str,
                    maxLikeCount: int,
                    sharedItemsFederatedDomains: [],
-                   signingPrivateKeyPem: str) -> str:
+                   signingPrivateKeyPem: str,
+                   CWlists: {}) -> str:
     """Show the moderation feed as html
     This is what you see when selecting the "mod" timeline
     """
@@ -72,7 +73,7 @@ def htmlModeration(cssCache: {}, defaultTimeline: str,
                         peertubeInstances, allowLocalNetworkAccess,
                         textModeBanner, accessKeys, systemLanguage,
                         maxLikeCount, sharedItemsFederatedDomains,
-                        signingPrivateKeyPem)
+                        signingPrivateKeyPem, CWlists)
 
 
 def htmlAccountInfo(cssCache: {}, translate: {},

webapp_post.py

@@ -77,6 +77,7 @@ from webfinger import webfingerHandle
 from speaker import updateSpeaker
 from languages import autoTranslatePost
 from blocking import isBlocked
+from blocking import addCWfromLists
 
 
 def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None:
@@ -1205,7 +1206,8 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
                          manuallyApprovesFollowers: bool,
                          showPublicOnly: bool,
                          storeToCache: bool,
-                         useCacheOnly: bool) -> str:
+                         useCacheOnly: bool,
+                         CWlists: {}) -> str:
     """ Shows a single post as html
     """
     if not postJsonObject:
@@ -1651,6 +1653,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str,
     if newFooterStr:
         footerStr = newFooterStr
 
+    # add any content warning from the cwlists directory
+    addCWfromLists(postJsonObject, CWlists, translate)
+
     postIsSensitive = False
     if postJsonObject['object'].get('sensitive'):
         # sensitive posts should have a summary
@@ -1825,7 +1830,8 @@ def htmlIndividualPost(cssCache: {},
                        peertubeInstances: [],
                        allowLocalNetworkAccess: bool,
                        themeName: str, systemLanguage: str,
-                       maxLikeCount: int, signingPrivateKeyPem: str) -> str:
+                       maxLikeCount: int, signingPrivateKeyPem: str,
+                       CWlists: {}) -> str:
     """Show an individual post as html
     """
     postStr = ''
@@ -1878,7 +1884,8 @@ def htmlIndividualPost(cssCache: {},
                              peertubeInstances,
                              allowLocalNetworkAccess, themeName,
                              systemLanguage, maxLikeCount,
-                             False, authorized, False, False, False, False)
+                             False, authorized, False, False, False, False,
+                             CWlists)
     messageId = removeIdEnding(postJsonObject['id'])
 
     # show the previous posts
@@ -1910,7 +1917,8 @@ def htmlIndividualPost(cssCache: {},
                                          themeName, systemLanguage,
                                          maxLikeCount,
                                          False, authorized,
-                                         False, False, False, False) + postStr
+                                         False, False, False, False,
+                                         CWlists) + postStr
 
     # show the following posts
     postFilename = locatePost(baseDir, nickname, domain, messageId)
@@ -1944,7 +1952,8 @@ def htmlIndividualPost(cssCache: {},
                                          themeName, systemLanguage,
                                          maxLikeCount,
                                          False, authorized,
-                                         False, False, False, False)
+                                         False, False, False, False,
+                                         CWlists)
     cssFilename = baseDir + '/epicyon-profile.css'
     if os.path.isfile(baseDir + '/epicyon.css'):
         cssFilename = baseDir + '/epicyon.css'
@@ -1968,7 +1977,7 @@ def htmlPostReplies(cssCache: {},
                     allowLocalNetworkAccess: bool,
                     themeName: str, systemLanguage: str,
                     maxLikeCount: int,
-                    signingPrivateKeyPem: str) -> str:
+                    signingPrivateKeyPem: str, CWlists: {}) -> str:
     """Show the replies to an individual post as html
     """
     repliesStr = ''
@@ -1991,7 +2000,8 @@ def htmlPostReplies(cssCache: {},
                                      allowLocalNetworkAccess,
                                      themeName, systemLanguage,
                                      maxLikeCount,
-                                     False, False, False, False, False, False)
+                                     False, False, False, False, False, False,
+                                     CWlists)
 
     cssFilename = baseDir + '/epicyon-profile.css'
     if os.path.isfile(baseDir + '/epicyon.css'):

webapp_profile.py

@@ -89,7 +89,8 @@ def htmlProfileAfterSearch(cssCache: {},
                            accessKeys: {},
                            systemLanguage: str,
                            maxLikeCount: int,
-                           signingPrivateKeyPem: str) -> str:
+                           signingPrivateKeyPem: str,
+                           CWlists: {}) -> str:
     """Show a profile page after a search for a fediverse address
     """
     http = False
@@ -306,7 +307,8 @@ def htmlProfileAfterSearch(cssCache: {},
                                      peertubeInstances,
                                      allowLocalNetworkAccess,
                                      themeName, systemLanguage, maxLikeCount,
-                                     False, False, False, False, False, False)
+                                     False, False, False, False, False, False,
+                                     CWlists)
             i += 1
             if i >= 8:
                 break
@@ -516,8 +518,9 @@ def htmlProfile(signingPrivateKeyPem: str,
                 debug: bool, accessKeys: {}, city: str,
                 systemLanguage: str, maxLikeCount: int,
                 sharedItemsFederatedDomains: [],
-                extraJson: {} = None, pageNumber: int = None,
+                extraJson: {}, pageNumber: int,
-                maxItemsPerPage: int = None) -> str:
+                maxItemsPerPage: int,
+                CWlists: {}) -> str:
     """Show the profile page as html
     """
     nickname = profileJson['preferredUsername']
@@ -539,8 +542,8 @@ def htmlProfile(signingPrivateKeyPem: str,
                                newswire, theme, extraJson,
                                allowLocalNetworkAccess, accessKeys,
                                systemLanguage, maxLikeCount,
-                               sharedItemsFederatedDomains,
+                               sharedItemsFederatedDomains, None,
-                               pageNumber, maxItemsPerPage)
+                               pageNumber, maxItemsPerPage, CWlists)
 
     domain, port = getDomainFromActor(profileJson['id'])
     if not domain:
@@ -897,7 +900,8 @@ def htmlProfile(signingPrivateKeyPem: str,
                               allowLocalNetworkAccess,
                               theme, systemLanguage,
                               maxLikeCount,
-                              signingPrivateKeyPem) + licenseStr
+                              signingPrivateKeyPem,
+                              CWlists) + licenseStr
     if not isGroup:
         if selected == 'following':
             profileStr += \
@@ -963,7 +967,8 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int,
                       allowLocalNetworkAccess: bool,
                       themeName: str, systemLanguage: str,
                       maxLikeCount: int,
-                      signingPrivateKeyPem: str) -> str:
+                      signingPrivateKeyPem: str,
+                      CWlists: {}) -> str:
     """Shows posts on the profile screen
     These should only be public posts
     """
@@ -1008,7 +1013,8 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int,
                                          themeName, systemLanguage,
                                          maxLikeCount,
                                          False, False, False,
-                                         True, False, False)
+                                         True, False, False,
+                                         CWlists)
                 if postStr:
                     profileStr += postStr + separatorStr
                     ctr += 1

webapp_search.py

@@ -598,7 +598,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str,
                       themeName: str, boxName: str,
                       systemLanguage: str,
                       maxLikeCount: int,
-                      signingPrivateKeyPem: str) -> str:
+                      signingPrivateKeyPem: str,
+                      CWlists: {}) -> str:
     """Show a page containing search results for your post history
     """
     if historysearch.startswith("'"):
@@ -683,7 +684,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str,
                                  themeName, systemLanguage, maxLikeCount,
                                  showIndividualPostIcons,
                                  showIndividualPostIcons,
-                                 False, False, False, False)
+                                 False, False, False, False,
+                                 CWlists)
         if postStr:
             historySearchForm += separatorStr + postStr
         index += 1
@@ -707,7 +709,8 @@ def htmlHashtagSearch(cssCache: {},
                       allowLocalNetworkAccess: bool,
                       themeName: str, systemLanguage: str,
                       maxLikeCount: int,
-                      signingPrivateKeyPem: str) -> str:
+                      signingPrivateKeyPem: str,
+                      CWlists: {}) -> str:
     """Show a page containing search results for a hashtag
     or after selecting a hashtag from the swarm
     """
@@ -865,7 +868,7 @@ def htmlHashtagSearch(cssCache: {},
                                  showRepeats, showIcons,
                                  manuallyApprovesFollowers,
                                  showPublicOnly,
-                                 storeToCache, False)
+                                 storeToCache, False, CWlists)
         if postStr:
             hashtagSearchForm += separatorStr + postStr
         index += 1

webapp_timeline.py

@@ -446,7 +446,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str,
                  accessKeys: {}, systemLanguage: str,
                  maxLikeCount: int,
                  sharedItemsFederatedDomains: [],
-                 signingPrivateKeyPem: str) -> str:
+                 signingPrivateKeyPem: str,
+                 CWlists: {}) -> str:
     """Show the timeline as html
     """
     enableTimingLog = False
@@ -913,7 +914,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str,
                                              boxName != 'dm',
                                              showIndividualPostIcons,
                                              manuallyApproveFollowers,
-                                             False, True, useCacheOnly)
+                                             False, True, useCacheOnly,
+                                             CWlists)
                     _logTimelineTiming(enableTimingLog,
                                        timelineStartTime, boxName, '12')
 
@@ -1137,7 +1139,8 @@ def htmlShares(cssCache: {}, defaultTimeline: str,
                accessKeys: {}, systemLanguage: str,
                maxLikeCount: int,
                sharedItemsFederatedDomains: [],
-               signingPrivateKeyPem: str) -> str:
+               signingPrivateKeyPem: str,
+               CWlists: {}) -> str:
     """Show the shares timeline as html
     """
     manuallyApproveFollowers = \
@@ -1162,7 +1165,8 @@ def htmlShares(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlWanted(cssCache: {}, defaultTimeline: str,
@@ -1189,7 +1193,8 @@ def htmlWanted(cssCache: {}, defaultTimeline: str,
                accessKeys: {}, systemLanguage: str,
                maxLikeCount: int,
                sharedItemsFederatedDomains: [],
-               signingPrivateKeyPem: str) -> str:
+               signingPrivateKeyPem: str,
+               CWlists: {}) -> str:
     """Show the wanted timeline as html
     """
     manuallyApproveFollowers = \
@@ -1214,7 +1219,8 @@ def htmlWanted(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInbox(cssCache: {}, defaultTimeline: str,
@@ -1242,7 +1248,8 @@ def htmlInbox(cssCache: {}, defaultTimeline: str,
               accessKeys: {}, systemLanguage: str,
               maxLikeCount: int,
               sharedItemsFederatedDomains: [],
-              signingPrivateKeyPem: str) -> str:
+              signingPrivateKeyPem: str,
+              CWlists: {}) -> str:
     """Show the inbox as html
     """
     manuallyApproveFollowers = \
@@ -1267,7 +1274,8 @@ def htmlInbox(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlBookmarks(cssCache: {}, defaultTimeline: str,
@@ -1295,7 +1303,8 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str,
                   accessKeys: {}, systemLanguage: str,
                   maxLikeCount: int,
                   sharedItemsFederatedDomains: [],
-                  signingPrivateKeyPem: str) -> str:
+                  signingPrivateKeyPem: str,
+                  CWlists: {}) -> str:
     """Show the bookmarks as html
     """
     manuallyApproveFollowers = \
@@ -1320,7 +1329,8 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxDMs(cssCache: {}, defaultTimeline: str,
@@ -1348,7 +1358,8 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str,
                  accessKeys: {}, systemLanguage: str,
                  maxLikeCount: int,
                  sharedItemsFederatedDomains: [],
-                 signingPrivateKeyPem: str) -> str:
+                 signingPrivateKeyPem: str,
+                 CWlists: {}) -> str:
     """Show the DM timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1368,7 +1379,8 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxReplies(cssCache: {}, defaultTimeline: str,
@@ -1396,7 +1408,8 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str,
                      accessKeys: {}, systemLanguage: str,
                      maxLikeCount: int,
                      sharedItemsFederatedDomains: [],
-                     signingPrivateKeyPem: str) -> str:
+                     signingPrivateKeyPem: str,
+                     CWlists: {}) -> str:
     """Show the replies timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1417,7 +1430,8 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxMedia(cssCache: {}, defaultTimeline: str,
@@ -1445,7 +1459,8 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str,
                    accessKeys: {}, systemLanguage: str,
                    maxLikeCount: int,
                    sharedItemsFederatedDomains: [],
-                   signingPrivateKeyPem: str) -> str:
+                   signingPrivateKeyPem: str,
+                   CWlists: {}) -> str:
     """Show the media timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1466,7 +1481,8 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxBlogs(cssCache: {}, defaultTimeline: str,
@@ -1494,7 +1510,8 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str,
                    accessKeys: {}, systemLanguage: str,
                    maxLikeCount: int,
                    sharedItemsFederatedDomains: [],
-                   signingPrivateKeyPem: str) -> str:
+                   signingPrivateKeyPem: str,
+                   CWlists: {}) -> str:
     """Show the blogs timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1515,7 +1532,8 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxFeatures(cssCache: {}, defaultTimeline: str,
@@ -1544,7 +1562,8 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str,
                       accessKeys: {}, systemLanguage: str,
                       maxLikeCount: int,
                       sharedItemsFederatedDomains: [],
-                      signingPrivateKeyPem: str) -> str:
+                      signingPrivateKeyPem: str,
+                      CWlists: {}) -> str:
     """Show the features timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1565,7 +1584,8 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlInboxNews(cssCache: {}, defaultTimeline: str,
@@ -1593,7 +1613,8 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str,
                   accessKeys: {}, systemLanguage: str,
                   maxLikeCount: int,
                   sharedItemsFederatedDomains: [],
-                  signingPrivateKeyPem: str) -> str:
+                  signingPrivateKeyPem: str,
+                  CWlists: {}) -> str:
     """Show the news timeline as html
     """
     return htmlTimeline(cssCache, defaultTimeline,
@@ -1614,7 +1635,8 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)
 
 
 def htmlOutbox(cssCache: {}, defaultTimeline: str,
@@ -1642,7 +1664,8 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str,
                accessKeys: {}, systemLanguage: str,
                maxLikeCount: int,
                sharedItemsFederatedDomains: [],
-               signingPrivateKeyPem: str) -> str:
+               signingPrivateKeyPem: str,
+               CWlists: {}) -> str:
     """Show the Outbox as html
     """
     manuallyApproveFollowers = \
@@ -1664,4 +1687,5 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str,
                         authorized, None, theme, peertubeInstances,
                         allowLocalNetworkAccess, textModeBanner,
                         accessKeys, systemLanguage, maxLikeCount,
-                        sharedItemsFederatedDomains, signingPrivateKeyPem)
+                        sharedItemsFederatedDomains, signingPrivateKeyPem,
+                        CWlists)