From 979bb79f7642a0e767387fb02e19c7858badfd5d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 20 Oct 2021 21:57:49 +0100 Subject: [PATCH 01/27] Exclude mS for better use of graph bar width --- fitnessFunctions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fitnessFunctions.py b/fitnessFunctions.py index 1a4bb1976..f3531e7aa 100644 --- a/fitnessFunctions.py +++ b/fitnessFunctions.py @@ -112,7 +112,7 @@ def htmlWatchPointsGraph(baseDir: str, fitness: {}, fitnessId: str, htmlStr += \ '\n' + \ ' ' + name + '\n' + \ - ' ' + str(timeMS) + 'mS\n' + \ + ' ' + str(timeMS) + '\n' + \ '\n' ctr += 1 if ctr >= maxEntries: From 1dd5b8d026c9e3865e7285ad9880a9f82acc1305 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 20 Oct 2021 22:06:00 +0100 Subject: [PATCH 02/27] Fewer watchpoints --- daemon.py | 50 ++------------------------------------------------ 1 file changed, 2 insertions(+), 48 deletions(-) diff --git a/daemon.py b/daemon.py index 5c6849af2..034047507 100644 --- a/daemon.py +++ b/daemon.py @@ -6854,9 +6854,6 @@ class PubServer(BaseHTTPRequestHandler): self._redirect_headers(actorPathStr, cookie, callingDomain) return - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton1', - self.server.debug) if not self.server.session: print('Starting new session during repeat button') self.server.session = createSession(proxyType) @@ -6866,9 +6863,6 @@ class PubServer(BaseHTTPRequestHandler): self._404() self.server.GETbusy = False return - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton2', - self.server.debug) self.server.actorRepeat = path.split('?actor=')[1] announceToStr = \ localActorUrl(httpPrefix, self.postToNickname, domainFull) + \ @@ -6891,9 +6885,6 @@ class PubServer(BaseHTTPRequestHandler): debug, self.server.projectVersion, self.server.signingPrivateKeyPem) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton3', - self.server.debug) announceFilename = None if announceJson: # save the announce straight to the outbox @@ -6905,10 +6896,6 @@ class PubServer(BaseHTTPRequestHandler): self.postToNickname, domainFull, announceJson, 'outbox') - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton4', - self.server.debug) - # clear the icon from the cache so that it gets updated if self.server.iconsCache.get('repeat.png'): del self.server.iconsCache['repeat.png'] @@ -6917,7 +6904,7 @@ class PubServer(BaseHTTPRequestHandler): self._postToOutboxThread(announceJson) fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton5', + '_GET', '_announceButton postToOutboxThread', self.server.debug) # generate the html for the announce @@ -6927,9 +6914,6 @@ class PubServer(BaseHTTPRequestHandler): cachedPostFilename = \ getCachedPostFilename(baseDir, self.postToNickname, domain, announceJson) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton6', - self.server.debug) if debug: print('Announced post json: ' + str(announceJson)) print('Announced post nickname: ' + @@ -6940,9 +6924,6 @@ class PubServer(BaseHTTPRequestHandler): followerApprovalActive(baseDir, self.postToNickname, domain) showRepeats = not isDM(announceJson) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton7', - self.server.debug) individualPostAsHtml(self.server.signingPrivateKeyPem, False, self.server.recentPostsCache, self.server.maxRecentPosts, @@ -6969,9 +6950,6 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_announceButton8', - self.server.debug) self.server.GETbusy = False actorAbsolute = self._getInstanceUrl(callingDomain) + actor @@ -7348,9 +7326,6 @@ class PubServer(BaseHTTPRequestHandler): self._redirect_headers(actorPathStr, cookie, callingDomain) return - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton1', - self.server.debug) if not self.server.session: print('Starting new session during like') self.server.session = createSession(proxyType) @@ -7359,9 +7334,6 @@ class PubServer(BaseHTTPRequestHandler): self._404() self.server.GETbusy = False return - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton2', - self.server.debug) likeActor = \ localActorUrl(httpPrefix, self.postToNickname, domainFull) actorLiked = path.split('?actor=')[1] @@ -7372,9 +7344,6 @@ class PubServer(BaseHTTPRequestHandler): origActor, origPostUrl, origFilename = \ getOriginalPostFromAnnounceUrl(likeUrl, baseDir, self.postToNickname, domain) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton3', - self.server.debug) likeUrl2 = likeUrl likedPostFilename = origFilename if origActor and origPostUrl: @@ -7394,7 +7363,7 @@ class PubServer(BaseHTTPRequestHandler): self._postToOutbox(likeJson, self.server.projectVersion, None) fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton4', + '_GET', '_likeButton postToOutbox', self.server.debug) print('Locating liked post ' + likeUrl) @@ -7403,9 +7372,6 @@ class PubServer(BaseHTTPRequestHandler): likedPostFilename = \ locatePost(baseDir, self.postToNickname, domain, likeUrl) if likedPostFilename: - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton5', - self.server.debug) recentPostsCache = self.server.recentPostsCache likedPostJson = loadJson(likedPostFilename, 0, 1) if origFilename and origPostUrl: @@ -7417,9 +7383,6 @@ class PubServer(BaseHTTPRequestHandler): likedPostFilename = origFilename if debug: print('Updating likes for ' + likedPostFilename) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton6', - self.server.debug) updateLikesCollection(recentPostsCache, baseDir, likedPostFilename, likeUrl, likeActor, self.postToNickname, domain, @@ -7428,9 +7391,6 @@ class PubServer(BaseHTTPRequestHandler): print('Regenerating html post for changed likes collection') # clear the icon from the cache so that it gets updated if likedPostJson: - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton6', - self.server.debug) cachedPostFilename = \ getCachedPostFilename(baseDir, self.postToNickname, domain, likedPostJson) @@ -7444,9 +7404,6 @@ class PubServer(BaseHTTPRequestHandler): followerApprovalActive(baseDir, self.postToNickname, domain) showRepeats = not isDM(likedPostJson) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton7', - self.server.debug) individualPostAsHtml(self.server.signingPrivateKeyPem, False, self.server.recentPostsCache, self.server.maxRecentPosts, @@ -7474,9 +7431,6 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False) - fitnessPerformance(GETstartTime, self.server.fitness, - '_GET', '_likeButton8', - self.server.debug) else: print('WARN: Liked post not found: ' + likedPostFilename) # clear the icon from the cache so that it gets updated From 8d3c476fca23e2def55bddbe2b76c31f3e92b11b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 11:38:29 +0100 Subject: [PATCH 03/27] Domain lists --- lists/murdoch.json | 170 ++++++++++++++++++++++++++++++++++++++++++++ lists/murdoch.json~ | 169 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 lists/murdoch.json create mode 100644 lists/murdoch.json~ diff --git a/lists/murdoch.json b/lists/murdoch.json new file mode 100644 index 000000000..44b2a406f --- /dev/null +++ b/lists/murdoch.json @@ -0,0 +1,170 @@ +{ + "name": "Murdoch press", + "warning": "Murdoch Press", + "domains": [ + api.news, + content.api.news, + newscdn.com.au, + resourcesssl.newscdn.com.au, + thesun.co.uk, + ads.thesun.co.uk, + cdn.thesun.co.uk, + cmp.cdn.thesun.co.uk, + www.thesun.co.uk, + thetimes.co.uk, + ads.thetimes.co.uk, + www.thetimes.co.uk, + thesundaytimes.co.uk, + pressassociation.com, + news.co.uk, + www.news.co.uk, + newscorpaustralia.com, + www.newscorpaustralia.com, + logflume.newscorpaustralia.com, + theaustralian.com.au, + media.theaustralian.com.au, + www.theaustralian.com.au, + aap.com.au, + www.aap.com.au, + news.com.au, + tags.news.com.au, + go.news.com.au, + origin.go.news.com.au, + www.news.com.au, + skynews.com.au, + www.skynews.com.au, + www.skyweather.com.au, + skyweather.com.au, + australiannewschannel.com.au, + weeklytimesnow.com.au, + www.weeklytimesnow.com.au, + dailytelegraph.com.au, + www.dailytelegraph.com.au, + heraldsun.com.au, + www.heraldsun.com.au, + geelongadvertiser.com.au, + www.geelongadvertiser.com.au, + couriermail.com.au, + www.couriermail.com.au, + thesundaymail.com.au, + goldcoastbulletin.com.au, + www.goldcoastbulletin.com.au, + cairnspost.com.au, + www.cairnspost.com.au, + townsvillebulletin.com.au, + www.townsvillebulletin.com.au, + adelaidenow.com.au, + www.adelaidenow.com.au, + themercury.com.au, + www.themercury.com.au, + ntnews.com.au, + www.ntnews.com.au, + postcourier.com.pg, + nypost.com, + embeds.nypost.com, + pagesix.com, + realtor.com, + wsj.com, + www.wsj.com, + accounts.wsj.com, + video-api.wsj.com, + wsj.net, + images.wsj.net, + m4.wsj.net, + m.wsj.net, + s.wsj.net, + si.wsj.net, + sts3.wsj.net, + vir.wsj.net, + foxnews.com, + www.foxnews.com, + a57.foxnews.com, + static.foxnews.com, + fncstatic.com, + global.fncstatic.com, + foxnewsgo.com, + www.foxbusiness.com, + fox.com, + strike.fox.com, + foxbusiness.com, + www.foxbusiness.com, + foxsports.com, + fssta.com, + b.fssta.com, + foxsports.com.au, + dowjones.com, + www.dowjones.com, + images.dowjones.com, + factiva.com, + barrons.com, + accounts.barrons.com, + asset.barrons.com, + images.barrons.com, + video-api.barrons.com, + www.barrons.com, + marketwatch.com, + www.marketwatch.com, + heatst.com, + fnlondon.com, + www.fnlondon.com, + assets.fnlondon.com, + images.fnlondon.com, + mansionglobal.com, + images.mansionglobal.com, + www.mansionglobal.com, + spindices.com, + spglobal.com, + www.spglobal.com, + talksport.com, + ads.talksport.com, + cdn259.talksport.com, + harpercollins.com, + www.harpercollins.com, + bestrecipes.com.au, + img.bestrecipes.com.au, + www.bestrecipes.com.au, + hipages.com.au, + img.hipages.com.au, + homeimprovementpages.com.au, + assets.homeimprovementpages.com.au, + odds.com.au, + www.odds.com.au, + onebigswitch.com.au, + suddenly.com.au, + www.suddenly.com.au, + supercoach.com.au, + punters.com.au, + www.punters.com.au, + kayosports.com.au, + foxtel.com.au, + www.foxtel.com.au, + newscorp.com, + storyful.com, + vogue.com.au, + media.vogue.com.au, + www.vogue.com.au, + taste.com.au, + img.taste.com.au, + www.taste.com.au, + kidspot.com.au, + go.kidspot.com.au, + origin.go.kidspot.com.au, + img.kidspot.com.au, + www.kidspot.com.au, + bodyandsoul.com.au, + go.bodyandsoul.com.au, + origin.bodyandsoul.com.au, + www.bodyandsoul.com.au, + realcommercial.com.au, + www.realcommercial.com.au, + reastatic.net, + au.reastatic.net, + rui.au.reastatic.net, + s1.rui.au.reastatic.net, + realestate.com.au, + www.realestate.com.au, + whereilive.com.au, + www.newsoftheworld.co.uk, + newsoftheworld.co.uk + ] +} diff --git a/lists/murdoch.json~ b/lists/murdoch.json~ new file mode 100644 index 000000000..87ef55f13 --- /dev/null +++ b/lists/murdoch.json~ @@ -0,0 +1,169 @@ +{ + "warning": "Murdoch Press", + "domains": [ + api.news + content.api.news + newscdn.com.au + resourcesssl.newscdn.com.au + thesun.co.uk + ads.thesun.co.uk + cdn.thesun.co.uk + cmp.cdn.thesun.co.uk + www.thesun.co.uk + thetimes.co.uk + ads.thetimes.co.uk + www.thetimes.co.uk + thesundaytimes.co.uk + pressassociation.com + news.co.uk + www.news.co.uk + newscorpaustralia.com + www.newscorpaustralia.com + logflume.newscorpaustralia.com + theaustralian.com.au + media.theaustralian.com.au + www.theaustralian.com.au + aap.com.au + www.aap.com.au + news.com.au + tags.news.com.au + go.news.com.au + origin.go.news.com.au + www.news.com.au + skynews.com.au + www.skynews.com.au + www.skyweather.com.au + skyweather.com.au + australiannewschannel.com.au + weeklytimesnow.com.au + www.weeklytimesnow.com.au + dailytelegraph.com.au + www.dailytelegraph.com.au + heraldsun.com.au + www.heraldsun.com.au + geelongadvertiser.com.au + www.geelongadvertiser.com.au + couriermail.com.au + www.couriermail.com.au + thesundaymail.com.au + goldcoastbulletin.com.au + www.goldcoastbulletin.com.au + cairnspost.com.au + www.cairnspost.com.au + townsvillebulletin.com.au + www.townsvillebulletin.com.au + adelaidenow.com.au + www.adelaidenow.com.au + themercury.com.au + www.themercury.com.au + ntnews.com.au + www.ntnews.com.au + postcourier.com.pg + nypost.com + embeds.nypost.com + pagesix.com + realtor.com + wsj.com + www.wsj.com + accounts.wsj.com + video-api.wsj.com + wsj.net + images.wsj.net + m4.wsj.net + m.wsj.net + s.wsj.net + si.wsj.net + sts3.wsj.net + vir.wsj.net + foxnews.com + www.foxnews.com + a57.foxnews.com + static.foxnews.com + fncstatic.com + global.fncstatic.com + foxnewsgo.com + www.foxbusiness.com + fox.com + strike.fox.com + foxbusiness.com + www.foxbusiness.com + foxsports.com + fssta.com + b.fssta.com + foxsports.com.au + dowjones.com + www.dowjones.com + images.dowjones.com + factiva.com + barrons.com + accounts.barrons.com + asset.barrons.com + images.barrons.com + video-api.barrons.com + www.barrons.com + marketwatch.com + www.marketwatch.com + heatst.com + fnlondon.com + www.fnlondon.com + assets.fnlondon.com + images.fnlondon.com + mansionglobal.com + images.mansionglobal.com + www.mansionglobal.com + spindices.com + spglobal.com + www.spglobal.com + talksport.com + ads.talksport.com + cdn259.talksport.com + harpercollins.com + www.harpercollins.com + bestrecipes.com.au + img.bestrecipes.com.au + www.bestrecipes.com.au + hipages.com.au + img.hipages.com.au + homeimprovementpages.com.au + assets.homeimprovementpages.com.au + odds.com.au + www.odds.com.au + onebigswitch.com.au + suddenly.com.au + www.suddenly.com.au + supercoach.com.au + punters.com.au + www.punters.com.au + kayosports.com.au + foxtel.com.au + www.foxtel.com.au + newscorp.com + storyful.com + vogue.com.au + media.vogue.com.au + www.vogue.com.au + taste.com.au + img.taste.com.au + www.taste.com.au + kidspot.com.au + go.kidspot.com.au + origin.go.kidspot.com.au + img.kidspot.com.au + www.kidspot.com.au + bodyandsoul.com.au + go.bodyandsoul.com.au + origin.bodyandsoul.com.au + www.bodyandsoul.com.au + realcommercial.com.au + www.realcommercial.com.au + reastatic.net + au.reastatic.net + rui.au.reastatic.net + s1.rui.au.reastatic.net + realestate.com.au + www.realestate.com.au + whereilive.com.au + www.newsoftheworld.co.uk + newsoftheworld.co.uk + ] +} From 13d1224b35be2da011e09e8f4a77bdf918d934ca Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 11:39:08 +0100 Subject: [PATCH 04/27] Tidying --- lists/murdoch.json~ | 169 -------------------------------------------- 1 file changed, 169 deletions(-) delete mode 100644 lists/murdoch.json~ diff --git a/lists/murdoch.json~ b/lists/murdoch.json~ deleted file mode 100644 index 87ef55f13..000000000 --- a/lists/murdoch.json~ +++ /dev/null @@ -1,169 +0,0 @@ -{ - "warning": "Murdoch Press", - "domains": [ - api.news - content.api.news - newscdn.com.au - resourcesssl.newscdn.com.au - thesun.co.uk - ads.thesun.co.uk - cdn.thesun.co.uk - cmp.cdn.thesun.co.uk - www.thesun.co.uk - thetimes.co.uk - ads.thetimes.co.uk - www.thetimes.co.uk - thesundaytimes.co.uk - pressassociation.com - news.co.uk - www.news.co.uk - newscorpaustralia.com - www.newscorpaustralia.com - logflume.newscorpaustralia.com - theaustralian.com.au - media.theaustralian.com.au - www.theaustralian.com.au - aap.com.au - www.aap.com.au - news.com.au - tags.news.com.au - go.news.com.au - origin.go.news.com.au - www.news.com.au - skynews.com.au - www.skynews.com.au - www.skyweather.com.au - skyweather.com.au - australiannewschannel.com.au - weeklytimesnow.com.au - www.weeklytimesnow.com.au - dailytelegraph.com.au - www.dailytelegraph.com.au - heraldsun.com.au - www.heraldsun.com.au - geelongadvertiser.com.au - www.geelongadvertiser.com.au - couriermail.com.au - www.couriermail.com.au - thesundaymail.com.au - goldcoastbulletin.com.au - www.goldcoastbulletin.com.au - cairnspost.com.au - www.cairnspost.com.au - townsvillebulletin.com.au - www.townsvillebulletin.com.au - adelaidenow.com.au - www.adelaidenow.com.au - themercury.com.au - www.themercury.com.au - ntnews.com.au - www.ntnews.com.au - postcourier.com.pg - nypost.com - embeds.nypost.com - pagesix.com - realtor.com - wsj.com - www.wsj.com - accounts.wsj.com - video-api.wsj.com - wsj.net - images.wsj.net - m4.wsj.net - m.wsj.net - s.wsj.net - si.wsj.net - sts3.wsj.net - vir.wsj.net - foxnews.com - www.foxnews.com - a57.foxnews.com - static.foxnews.com - fncstatic.com - global.fncstatic.com - foxnewsgo.com - www.foxbusiness.com - fox.com - strike.fox.com - foxbusiness.com - www.foxbusiness.com - foxsports.com - fssta.com - b.fssta.com - foxsports.com.au - dowjones.com - www.dowjones.com - images.dowjones.com - factiva.com - barrons.com - accounts.barrons.com - asset.barrons.com - images.barrons.com - video-api.barrons.com - www.barrons.com - marketwatch.com - www.marketwatch.com - heatst.com - fnlondon.com - www.fnlondon.com - assets.fnlondon.com - images.fnlondon.com - mansionglobal.com - images.mansionglobal.com - www.mansionglobal.com - spindices.com - spglobal.com - www.spglobal.com - talksport.com - ads.talksport.com - cdn259.talksport.com - harpercollins.com - www.harpercollins.com - bestrecipes.com.au - img.bestrecipes.com.au - www.bestrecipes.com.au - hipages.com.au - img.hipages.com.au - homeimprovementpages.com.au - assets.homeimprovementpages.com.au - odds.com.au - www.odds.com.au - onebigswitch.com.au - suddenly.com.au - www.suddenly.com.au - supercoach.com.au - punters.com.au - www.punters.com.au - kayosports.com.au - foxtel.com.au - www.foxtel.com.au - newscorp.com - storyful.com - vogue.com.au - media.vogue.com.au - www.vogue.com.au - taste.com.au - img.taste.com.au - www.taste.com.au - kidspot.com.au - go.kidspot.com.au - origin.go.kidspot.com.au - img.kidspot.com.au - www.kidspot.com.au - bodyandsoul.com.au - go.bodyandsoul.com.au - origin.bodyandsoul.com.au - www.bodyandsoul.com.au - realcommercial.com.au - www.realcommercial.com.au - reastatic.net - au.reastatic.net - rui.au.reastatic.net - s1.rui.au.reastatic.net - realestate.com.au - www.realestate.com.au - whereilive.com.au - www.newsoftheworld.co.uk - newsoftheworld.co.uk - ] -} From b8c957eb3824bc39e3cde15778854bd76f24b094 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 12:13:24 +0100 Subject: [PATCH 05/27] Load cw lists at startup --- blocking.py | 22 ++++++++++++++++++++++ daemon.py | 2 ++ 2 files changed, 24 insertions(+) diff --git a/blocking.py b/blocking.py index 5c70ca676..da372f8af 100644 --- a/blocking.py +++ b/blocking.py @@ -874,3 +874,25 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool: print('Broch mode has elapsed') return True return False + + +def loadLists(baseDir: str, verbose: bool) -> {}: + """Load lists used for blocking or warnings + """ + if not os.path.isdir(baseDir + '/lists'): + return {} + result = {} + for subdir, dirs, files in os.walk(baseDir + '/lists'): + for f in files: + if not f.endswith('.json'): + continue + listFilename = os.path.join(baseDir + '/lists', f) + listJson = loadJson(listFilename) + if not listJson: + continue + if listJson.get('name') and listJson.get('domains'): + name = listJson['name'] + if verbose: + print('List: ' + name) + result[name] = listJson + return result diff --git a/daemon.py b/daemon.py index 034047507..c382f1481 100644 --- a/daemon.py +++ b/daemon.py @@ -125,6 +125,7 @@ from media import replaceTwitter from media import attachMedia from media import pathIsVideo from media import pathIsAudio +from blocking import loadLists from blocking import updateBlockedCache from blocking import mutePost from blocking import unmutePost @@ -17164,6 +17165,7 @@ def runDaemon(defaultReplyIntervalHours: int, updateBlockedCache(baseDir, httpd.blockedCache, httpd.blockedCacheLastUpdated, httpd.blockedCacheUpdateSecs) + httpd.lists = loadLists(baseDir, True) # cache to store css files httpd.cssCache = {} From d16942976b608f1970c734356e3c977da6849bd0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 12:17:26 +0100 Subject: [PATCH 06/27] cw lists can contain words and domains --- blocking.py | 13 ++++++++----- lists/murdoch.json | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/blocking.py b/blocking.py index da372f8af..3ae7f96f5 100644 --- a/blocking.py +++ b/blocking.py @@ -890,9 +890,12 @@ def loadLists(baseDir: str, verbose: bool) -> {}: listJson = loadJson(listFilename) if not listJson: continue - if listJson.get('name') and listJson.get('domains'): - name = listJson['name'] - if verbose: - print('List: ' + name) - result[name] = listJson + if not listJson.get('name'): + continue + if not listJson('words') and not listJson.get('domains'): + continue + name = listJson['name'] + if verbose: + print('List: ' + name) + result[name] = listJson return result diff --git a/lists/murdoch.json b/lists/murdoch.json index 44b2a406f..31c4a9d5c 100644 --- a/lists/murdoch.json +++ b/lists/murdoch.json @@ -1,6 +1,7 @@ { "name": "Murdoch press", "warning": "Murdoch Press", + "words": [], "domains": [ api.news, content.api.news, From 18d1c54feabcc00bcdf5f682689cd376db39d876 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 14:08:21 +0100 Subject: [PATCH 07/27] Apply cw from lists --- blocking.py | 69 +++++++++++++++-- cwlists/murdoch.json | 171 ++++++++++++++++++++++++++++++++++++++++++ daemon.py | 111 +++++++++++++++++---------- inbox.py | 48 ++++++------ lists/murdoch.json | 171 ------------------------------------------ outbox.py | 5 +- schedule.py | 3 +- tests.py | 43 +++++++++++ webapp_confirm.py | 6 +- webapp_frontscreen.py | 15 ++-- webapp_moderation.py | 5 +- webapp_post.py | 24 ++++-- webapp_profile.py | 24 +++--- webapp_search.py | 11 ++- webapp_timeline.py | 72 ++++++++++++------ 15 files changed, 484 insertions(+), 294 deletions(-) create mode 100644 cwlists/murdoch.json delete mode 100644 lists/murdoch.json diff --git a/blocking.py b/blocking.py index 3ae7f96f5..f97f52549 100644 --- a/blocking.py +++ b/blocking.py @@ -876,26 +876,81 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool: return False -def loadLists(baseDir: str, verbose: bool) -> {}: - """Load lists used for blocking or warnings +def loadCWLists(baseDir: str, verbose: bool) -> {}: + """Load lists used for content warnings """ - if not os.path.isdir(baseDir + '/lists'): + if not os.path.isdir(baseDir + '/cwlists'): return {} result = {} - for subdir, dirs, files in os.walk(baseDir + '/lists'): + for subdir, dirs, files in os.walk(baseDir + '/cwlists'): for f in files: if not f.endswith('.json'): continue - listFilename = os.path.join(baseDir + '/lists', f) - listJson = loadJson(listFilename) + listFilename = os.path.join(baseDir + '/cwlists', f) + print('listFilename: ' + listFilename) + listJson = loadJson(listFilename, 0, 1) if not listJson: continue if not listJson.get('name'): continue - if not listJson('words') and not listJson.get('domains'): + if not listJson.get('words') and not listJson.get('domains'): continue name = listJson['name'] if verbose: print('List: ' + name) result[name] = listJson return result + + +def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}) -> None: + """Adds content warnings by matching the post content + against domains or keywords + """ + if not postJsonObject['object'].get('content'): + return + cw = '' + if postJsonObject['object'].get('summary'): + cw = postJsonObject['object']['summary'] + + content = postJsonObject['object']['content'] + for name, item in CWlists.items(): + if not item.get('warning'): + continue + warning = item['warning'] + + # is there a translated version of the warning? + if translate.get(warning): + warning = translate[warning] + + # is the warning already in the CW? + if warning in cw: + continue + + matched = False + + # match domains within the content + if item.get('domains'): + for domain in item['domains']: + if domain in content: + if cw: + cw = warning + ' / ' + cw + else: + cw = warning + matched = True + break + + if matched: + continue + + # match words within the content + if item.get('words'): + for wordStr in item['words']: + if wordStr in content: + if cw: + cw = warning + ' / ' + cw + else: + cw = warning + break + if cw: + postJsonObject['object']['summary'] = cw + postJsonObject['object']['sensitive'] = True diff --git a/cwlists/murdoch.json b/cwlists/murdoch.json new file mode 100644 index 000000000..52a4447ea --- /dev/null +++ b/cwlists/murdoch.json @@ -0,0 +1,171 @@ +{ + "name": "Murdoch press", + "warning": "Murdoch Press", + "words": [], + "domains": [ + "api.news", + "content.api.news", + "newscdn.com.au", + "resourcesssl.newscdn.com.au", + "thesun.co.uk", + "ads.thesun.co.uk", + "cdn.thesun.co.uk", + "cmp.cdn.thesun.co.uk", + "www.thesun.co.uk", + "thetimes.co.uk", + "ads.thetimes.co.uk", + "www.thetimes.co.uk", + "thesundaytimes.co.uk", + "pressassociation.com", + "news.co.uk", + "www.news.co.uk", + "newscorpaustralia.com", + "www.newscorpaustralia.com", + "logflume.newscorpaustralia.com", + "theaustralian.com.au", + "media.theaustralian.com.au", + "www.theaustralian.com.au", + "aap.com.au", + "www.aap.com.au", + "news.com.au", + "tags.news.com.au", + "go.news.com.au", + "origin.go.news.com.au", + "www.news.com.au", + "skynews.com.au", + "www.skynews.com.au", + "www.skyweather.com.au", + "skyweather.com.au", + "australiannewschannel.com.au", + "weeklytimesnow.com.au", + "www.weeklytimesnow.com.au", + "dailytelegraph.com.au", + "www.dailytelegraph.com.au", + "heraldsun.com.au", + "www.heraldsun.com.au", + "geelongadvertiser.com.au", + "www.geelongadvertiser.com.au", + "couriermail.com.au", + "www.couriermail.com.au", + "thesundaymail.com.au", + "goldcoastbulletin.com.au", + "www.goldcoastbulletin.com.au", + "cairnspost.com.au", + "www.cairnspost.com.au", + "townsvillebulletin.com.au", + "www.townsvillebulletin.com.au", + "adelaidenow.com.au", + "www.adelaidenow.com.au", + "themercury.com.au", + "www.themercury.com.au", + "ntnews.com.au", + "www.ntnews.com.au", + "postcourier.com.pg", + "nypost.com", + "embeds.nypost.com", + "pagesix.com", + "realtor.com", + "wsj.com", + "www.wsj.com", + "accounts.wsj.com", + "video-api.wsj.com", + "wsj.net", + "images.wsj.net", + "m4.wsj.net", + "m.wsj.net", + "s.wsj.net", + "si.wsj.net", + "sts3.wsj.net", + "vir.wsj.net", + "foxnews.com", + "www.foxnews.com", + "a57.foxnews.com", + "static.foxnews.com", + "fncstatic.com", + "global.fncstatic.com", + "foxnewsgo.com", + "www.foxbusiness.com", + "fox.com", + "strike.fox.com", + "foxbusiness.com", + "www.foxbusiness.com", + "foxsports.com", + "fssta.com", + "b.fssta.com", + "foxsports.com.au", + "dowjones.com", + "www.dowjones.com", + "images.dowjones.com", + "factiva.com", + "barrons.com", + "accounts.barrons.com", + "asset.barrons.com", + "images.barrons.com", + "video-api.barrons.com", + "www.barrons.com", + "marketwatch.com", + "www.marketwatch.com", + "heatst.com", + "fnlondon.com", + "www.fnlondon.com", + "assets.fnlondon.com", + "images.fnlondon.com", + "mansionglobal.com", + "images.mansionglobal.com", + "www.mansionglobal.com", + "spindices.com", + "spglobal.com", + "www.spglobal.com", + "talksport.com", + "ads.talksport.com", + "cdn259.talksport.com", + "harpercollins.com", + "www.harpercollins.com", + "bestrecipes.com.au", + "img.bestrecipes.com.au", + "www.bestrecipes.com.au", + "hipages.com.au", + "img.hipages.com.au", + "homeimprovementpages.com.au", + "assets.homeimprovementpages.com.au", + "odds.com.au", + "www.odds.com.au", + "onebigswitch.com.au", + "suddenly.com.au", + "www.suddenly.com.au", + "supercoach.com.au", + "punters.com.au", + "www.punters.com.au", + "kayosports.com.au", + "foxtel.com.au", + "www.foxtel.com.au", + "newscorp.com", + "storyful.com", + "vogue.com.au", + "media.vogue.com.au", + "www.vogue.com.au", + "taste.com.au", + "img.taste.com.au", + "www.taste.com.au", + "kidspot.com.au", + "go.kidspot.com.au", + "origin.go.kidspot.com.au", + "img.kidspot.com.au", + "www.kidspot.com.au", + "bodyandsoul.com.au", + "go.bodyandsoul.com.au", + "origin.bodyandsoul.com.au", + "www.bodyandsoul.com.au", + "realcommercial.com.au", + "www.realcommercial.com.au", + "reastatic.net", + "au.reastatic.net", + "rui.au.reastatic.net", + "s1.rui.au.reastatic.net", + "realestate.com.au", + "www.realestate.com.au", + "whereilive.com.au", + "www.newsoftheworld.co.uk", + "newsoftheworld.co.uk" + ] +} diff --git a/daemon.py b/daemon.py index c382f1481..d5e2af85e 100644 --- a/daemon.py +++ b/daemon.py @@ -125,7 +125,7 @@ from media import replaceTwitter from media import attachMedia from media import pathIsVideo from media import pathIsAudio -from blocking import loadLists +from blocking import loadCWLists from blocking import updateBlockedCache from blocking import mutePost from blocking import unmutePost @@ -1230,7 +1230,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.themeName, self.server.maxLikeCount, - self.server.maxRecentPosts) + self.server.maxRecentPosts, + self.server.CWlists) def _postToOutboxThread(self, messageJson: {}) -> bool: """Creates a thread to send a post @@ -3008,7 +3009,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -3066,7 +3068,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, 'outbox', self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if historyStr: msg = historyStr.encode('utf-8') msglen = len(msg) @@ -3104,7 +3107,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, 'bookmarks', self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if bookmarksStr: msg = bookmarksStr.encode('utf-8') msglen = len(msg) @@ -3208,7 +3212,8 @@ class PubServer(BaseHTTPRequestHandler): accessKeys, self.server.systemLanguage, self.server.maxLikeCount, - signingPrivateKeyPem) + signingPrivateKeyPem, + self.server.CWlists) if profileStr: msg = profileStr.encode('utf-8') msglen = len(msg) @@ -6728,7 +6733,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -6950,7 +6956,8 @@ class PubServer(BaseHTTPRequestHandler): showRepeats, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, + self.server.CWlists) self.server.GETbusy = False actorAbsolute = self._getInstanceUrl(callingDomain) + actor @@ -7431,7 +7438,8 @@ class PubServer(BaseHTTPRequestHandler): showRepeats, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, + self.server.CWlists) else: print('WARN: Liked post not found: ' + likedPostFilename) # clear the icon from the cache so that it gets updated @@ -7591,7 +7599,8 @@ class PubServer(BaseHTTPRequestHandler): showRepeats, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, + self.server.CWlists) else: print('WARN: Unliked post not found: ' + likedPostFilename) # clear the icon from the cache so that it gets updated @@ -7726,7 +7735,8 @@ class PubServer(BaseHTTPRequestHandler): showRepeats, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, + self.server.CWlists) else: print('WARN: Bookmarked post not found: ' + bookmarkFilename) # self._postToOutbox(bookmarkJson, self.server.projectVersion, None) @@ -7860,7 +7870,8 @@ class PubServer(BaseHTTPRequestHandler): showRepeats, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, + self.server.CWlists) else: print('WARN: Unbookmarked post not found: ' + bookmarkFilename) self.server.GETbusy = False @@ -7962,7 +7973,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if deleteStr: deleteStrLen = len(deleteStr) self._set_headers('text/html', deleteStrLen, @@ -8070,7 +8082,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, showPublicOnly, storeToCache, - useCacheOnly) + useCacheOnly, + self.server.CWlists) else: print('WARN: Muted post not found: ' + muteFilename) @@ -8179,7 +8192,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, showPublicOnly, storeToCache, - useCacheOnly) + useCacheOnly, + self.server.CWlists) else: print('WARN: Unmuted post not found: ' + muteFilename) self.server.GETbusy = False @@ -8302,7 +8316,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8401,7 +8416,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8510,7 +8526,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, rolesList, - None, None) + None, None, self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8623,7 +8639,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, skills, - None, None) + None, None, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8772,7 +8789,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.systemLanguage, self.server.maxLikeCount, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9018,7 +9036,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) if GETstartTime: fitnessPerformance(GETstartTime, self.server.fitness, @@ -9166,7 +9185,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9307,7 +9327,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9447,7 +9468,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9587,7 +9609,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9736,7 +9759,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9883,7 +9907,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9989,7 +10014,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10073,7 +10099,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10194,7 +10221,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10332,7 +10360,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10461,7 +10490,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, sharedItemsFederatedDomains, - self.server.signingPrivateKeyPem) + self.server.signingPrivateKeyPem, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10588,7 +10618,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, shares, - pageNumber, sharesPerPage) + pageNumber, sharesPerPage, + self.server.CWlists) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10712,7 +10743,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.sharedItemsFederatedDomains, following, pageNumber, - followsPerPage).encode('utf-8') + followsPerPage, + self.server.CWlists).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -10835,7 +10867,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.sharedItemsFederatedDomains, followers, pageNumber, - followsPerPage).encode('utf-8') + followsPerPage, + self.server.CWlists).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -10974,7 +11007,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, - None, None).encode('utf-8') + None, None, None, + self.server.CWlists).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -17165,7 +17199,7 @@ def runDaemon(defaultReplyIntervalHours: int, updateBlockedCache(baseDir, httpd.blockedCache, httpd.blockedCacheLastUpdated, httpd.blockedCacheUpdateSecs) - httpd.lists = loadLists(baseDir, True) + httpd.CWlists = loadCWLists(baseDir, True) # cache to store css files httpd.cssCache = {} @@ -17315,7 +17349,8 @@ def runDaemon(defaultReplyIntervalHours: int, httpd.systemLanguage, httpd.maxLikeCount, httpd.signingPrivateKeyPem, - httpd.defaultReplyIntervalHours), daemon=True) + httpd.defaultReplyIntervalHours, + httpd.CWlists), daemon=True) print('Creating scheduled post thread') httpd.thrPostSchedule = \ diff --git a/inbox.py b/inbox.py index 47dd760aa..4b1fd21c4 100644 --- a/inbox.py +++ b/inbox.py @@ -274,7 +274,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> None: + signingPrivateKeyPem: str, + CWlists: {}) -> None: """Converts the json post into html and stores it in a cache This enables the post to be quickly displayed later """ @@ -298,7 +299,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, showPublishedDateOnly, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - notDM, True, True, False, True, False) + notDM, True, True, False, True, False, + CWlists) def validInbox(baseDir: str, nickname: str, domain: str) -> bool: @@ -980,7 +982,7 @@ def _receiveLike(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int) -> bool: + maxLikeCount: int, CWlists: {}) -> bool: """Receives a Like activity within the POST section of HTTPServer """ if messageJson['type'] != 'Like': @@ -1082,7 +1084,7 @@ def _receiveLike(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, CWlists) return True @@ -1100,7 +1102,7 @@ def _receiveUndoLike(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int) -> bool: + maxLikeCount: int, CWlists: {}) -> bool: """Receives an undo like activity within the POST section of HTTPServer """ if messageJson['type'] != 'Undo': @@ -1191,7 +1193,7 @@ def _receiveUndoLike(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, CWlists) return True @@ -1208,7 +1210,7 @@ def _receiveBookmark(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int) -> bool: + maxLikeCount: int, CWlists: {}) -> bool: """Receives a bookmark activity within the POST section of HTTPServer """ if not messageJson.get('type'): @@ -1299,7 +1301,7 @@ def _receiveBookmark(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, CWlists) return True @@ -1316,7 +1318,7 @@ def _receiveUndoBookmark(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int) -> bool: + maxLikeCount: int, CWlists: {}) -> bool: """Receives an undo bookmark activity within the POST section of HTTPServer """ if not messageJson.get('type'): @@ -1408,7 +1410,7 @@ def _receiveUndoBookmark(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, CWlists) return True @@ -1502,7 +1504,7 @@ def _receiveAnnounce(recentPostsCache: {}, maxRecentPosts: int, allowDeletion: bool, peertubeInstances: [], - maxLikeCount: int) -> bool: + maxLikeCount: int, CWlists: {}) -> bool: """Receives an announce activity within the POST section of HTTPServer """ if messageJson['type'] != 'Announce': @@ -1614,7 +1616,7 @@ def _receiveAnnounce(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False) + False, True, False, CWlists) if not announceHtml: print('WARN: Unable to generate html for announce ' + str(messageJson)) @@ -2551,7 +2553,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - defaultReplyIntervalHours: int) -> bool: + defaultReplyIntervalHours: int, + CWlists: {}) -> bool: """ Anything which needs to be done after initial checks have passed """ actor = keyId @@ -2581,7 +2584,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount): + maxLikeCount, CWlists): if debug: print('DEBUG: Like accepted from ' + actor) return False @@ -2603,7 +2606,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount): + maxLikeCount, CWlists): if debug: print('DEBUG: Undo like accepted from ' + actor) return False @@ -2625,7 +2628,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount): + maxLikeCount, CWlists): if debug: print('DEBUG: Bookmark accepted from ' + actor) return False @@ -2647,7 +2650,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount): + maxLikeCount, CWlists): if debug: print('DEBUG: Undo bookmark accepted from ' + actor) return False @@ -2673,7 +2676,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, maxRecentPosts, allowDeletion, peertubeInstances, - maxLikeCount): + maxLikeCount, CWlists): if debug: print('DEBUG: Announce accepted from ' + actor) @@ -2954,7 +2957,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - signingPrivateKeyPem) + signingPrivateKeyPem, + CWlists) if debug: timeDiff = \ str(int((time.time() - htmlCacheStartTime) * @@ -3239,7 +3243,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, verifyAllSignatures: bool, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - defaultReplyIntervalHours: int) -> None: + defaultReplyIntervalHours: int, + CWlists: {}) -> None: """Processes received items and moves them to the appropriate directories """ @@ -3656,7 +3661,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, themeName, systemLanguage, maxLikeCount, signingPrivateKeyPem, - defaultReplyIntervalHours) + defaultReplyIntervalHours, + CWlists) if debug: pprint(queueJson['post']) print('Queue: Queue post accepted') diff --git a/lists/murdoch.json b/lists/murdoch.json deleted file mode 100644 index 31c4a9d5c..000000000 --- a/lists/murdoch.json +++ /dev/null @@ -1,171 +0,0 @@ -{ - "name": "Murdoch press", - "warning": "Murdoch Press", - "words": [], - "domains": [ - api.news, - content.api.news, - newscdn.com.au, - resourcesssl.newscdn.com.au, - thesun.co.uk, - ads.thesun.co.uk, - cdn.thesun.co.uk, - cmp.cdn.thesun.co.uk, - www.thesun.co.uk, - thetimes.co.uk, - ads.thetimes.co.uk, - www.thetimes.co.uk, - thesundaytimes.co.uk, - pressassociation.com, - news.co.uk, - www.news.co.uk, - newscorpaustralia.com, - www.newscorpaustralia.com, - logflume.newscorpaustralia.com, - theaustralian.com.au, - media.theaustralian.com.au, - www.theaustralian.com.au, - aap.com.au, - www.aap.com.au, - news.com.au, - tags.news.com.au, - go.news.com.au, - origin.go.news.com.au, - www.news.com.au, - skynews.com.au, - www.skynews.com.au, - www.skyweather.com.au, - skyweather.com.au, - australiannewschannel.com.au, - weeklytimesnow.com.au, - www.weeklytimesnow.com.au, - dailytelegraph.com.au, - www.dailytelegraph.com.au, - heraldsun.com.au, - www.heraldsun.com.au, - geelongadvertiser.com.au, - www.geelongadvertiser.com.au, - couriermail.com.au, - www.couriermail.com.au, - thesundaymail.com.au, - goldcoastbulletin.com.au, - www.goldcoastbulletin.com.au, - cairnspost.com.au, - www.cairnspost.com.au, - townsvillebulletin.com.au, - www.townsvillebulletin.com.au, - adelaidenow.com.au, - www.adelaidenow.com.au, - themercury.com.au, - www.themercury.com.au, - ntnews.com.au, - www.ntnews.com.au, - postcourier.com.pg, - nypost.com, - embeds.nypost.com, - pagesix.com, - realtor.com, - wsj.com, - www.wsj.com, - accounts.wsj.com, - video-api.wsj.com, - wsj.net, - images.wsj.net, - m4.wsj.net, - m.wsj.net, - s.wsj.net, - si.wsj.net, - sts3.wsj.net, - vir.wsj.net, - foxnews.com, - www.foxnews.com, - a57.foxnews.com, - static.foxnews.com, - fncstatic.com, - global.fncstatic.com, - foxnewsgo.com, - www.foxbusiness.com, - fox.com, - strike.fox.com, - foxbusiness.com, - www.foxbusiness.com, - foxsports.com, - fssta.com, - b.fssta.com, - foxsports.com.au, - dowjones.com, - www.dowjones.com, - images.dowjones.com, - factiva.com, - barrons.com, - accounts.barrons.com, - asset.barrons.com, - images.barrons.com, - video-api.barrons.com, - www.barrons.com, - marketwatch.com, - www.marketwatch.com, - heatst.com, - fnlondon.com, - www.fnlondon.com, - assets.fnlondon.com, - images.fnlondon.com, - mansionglobal.com, - images.mansionglobal.com, - www.mansionglobal.com, - spindices.com, - spglobal.com, - www.spglobal.com, - talksport.com, - ads.talksport.com, - cdn259.talksport.com, - harpercollins.com, - www.harpercollins.com, - bestrecipes.com.au, - img.bestrecipes.com.au, - www.bestrecipes.com.au, - hipages.com.au, - img.hipages.com.au, - homeimprovementpages.com.au, - assets.homeimprovementpages.com.au, - odds.com.au, - www.odds.com.au, - onebigswitch.com.au, - suddenly.com.au, - www.suddenly.com.au, - supercoach.com.au, - punters.com.au, - www.punters.com.au, - kayosports.com.au, - foxtel.com.au, - www.foxtel.com.au, - newscorp.com, - storyful.com, - vogue.com.au, - media.vogue.com.au, - www.vogue.com.au, - taste.com.au, - img.taste.com.au, - www.taste.com.au, - kidspot.com.au, - go.kidspot.com.au, - origin.go.kidspot.com.au, - img.kidspot.com.au, - www.kidspot.com.au, - bodyandsoul.com.au, - go.bodyandsoul.com.au, - origin.bodyandsoul.com.au, - www.bodyandsoul.com.au, - realcommercial.com.au, - www.realcommercial.com.au, - reastatic.net, - au.reastatic.net, - rui.au.reastatic.net, - s1.rui.au.reastatic.net, - realestate.com.au, - www.realestate.com.au, - whereilive.com.au, - www.newsoftheworld.co.uk, - newsoftheworld.co.uk - ] -} diff --git a/outbox.py b/outbox.py index 20c719db6..4db77d0cd 100644 --- a/outbox.py +++ b/outbox.py @@ -196,7 +196,7 @@ def postMessageToOutbox(session, translate: {}, signingPrivateKeyPem: str, peertubeInstances: str, theme: str, maxLikeCount: int, - maxRecentPosts: int) -> bool: + maxRecentPosts: int, CWlists: {}) -> bool: """post is received by the outbox Client to server message post https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery @@ -467,7 +467,8 @@ def postMessageToOutbox(session, translate: {}, boxNameIndex != 'dm', showIndividualPostIcons, manuallyApproveFollowers, - False, True, useCacheOnly) + False, True, useCacheOnly, + CWlists) if outboxAnnounce(recentPostsCache, baseDir, messageJson, debug): diff --git a/schedule.py b/schedule.py index 112bbfeef..c7a508d4a 100644 --- a/schedule.py +++ b/schedule.py @@ -124,7 +124,8 @@ def _updatePostSchedule(baseDir: str, handle: str, httpd, httpd.peertubeInstances, httpd.themeName, httpd.maxLikeCount, - httpd.maxRecentPosts): + httpd.maxRecentPosts, + httpd.CWlists): indexLines.remove(line) try: os.remove(postFilename) diff --git a/tests.py b/tests.py index 126ddb5ea..addec4342 100644 --- a/tests.py +++ b/tests.py @@ -164,6 +164,8 @@ from shares import updateSharedItemFederationToken from shares import mergeSharedItemTokens from shares import sendShareViaServer from shares import getSharedItemsCatalogViaServer +from blocking import loadCWLists +from blocking import addCWfromLists testServerGroupRunning = False testServerAliceRunning = False @@ -5764,6 +5766,46 @@ def _testWordsSimilarity() -> None: assert similarity > 80 +def _testAddCWfromLists(baseDir: str) -> None: + print('testAddCWfromLists') + translate = {} + CWlists = loadCWLists(baseDir, True) + assert CWlists + + postJsonObject = { + "object": { + "sensitive": False, + "summary": None, + "content": "" + } + } + addCWfromLists(postJsonObject, CWlists, translate) + assert postJsonObject['object']['sensitive'] is False + assert postJsonObject['object']['summary'] is None + + postJsonObject = { + "object": { + "sensitive": False, + "summary": None, + "content": "Blah blah news.co.uk blah blah" + } + } + addCWfromLists(postJsonObject, CWlists, translate) + assert postJsonObject['object']['sensitive'] is True + assert postJsonObject['object']['summary'] == "Murdoch Press" + + postJsonObject = { + "object": { + "sensitive": True, + "summary": "Existing CW", + "content": "Blah blah news.co.uk blah blah" + } + } + addCWfromLists(postJsonObject, CWlists, translate) + assert postJsonObject['object']['sensitive'] is True + assert postJsonObject['object']['summary'] == "Murdoch Press / Existing CW" + + def runAllTests(): baseDir = os.getcwd() print('Running tests...') @@ -5771,6 +5813,7 @@ def runAllTests(): _translateOntology(baseDir) _testGetPriceFromString() _testFunctions() + _testAddCWfromLists(baseDir) _testWordsSimilarity() _testSecondsBetweenPublished() _testSignAndVerify() diff --git a/webapp_confirm.py b/webapp_confirm.py index b24ce5422..e51db2bd7 100644 --- a/webapp_confirm.py +++ b/webapp_confirm.py @@ -35,7 +35,8 @@ def htmlConfirmDelete(cssCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, signingPrivateKeyPem: str) -> str: + maxLikeCount: int, signingPrivateKeyPem: str, + CWlists: {}) -> str: """Shows a screen asking to confirm the deletion of a post """ if '/statuses/' not in messageId: @@ -79,7 +80,8 @@ def htmlConfirmDelete(cssCache: {}, showPublishedDateOnly, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - False, False, False, False, False, False) + False, False, False, False, False, False, + CWlists) deletePostStr += '
' deletePostStr += \ '

' + \ diff --git a/webapp_frontscreen.py b/webapp_frontscreen.py index 8462d80e2..e53ac27b3 100644 --- a/webapp_frontscreen.py +++ b/webapp_frontscreen.py @@ -35,7 +35,7 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, CWlists: {}) -> str: """Shows posts on the front screen of a news instance These should only be public blog posts from the features timeline which is the blog timeline of the news actor @@ -81,7 +81,8 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, themeName, systemLanguage, maxLikeCount, False, False, False, - True, False, False) + True, False, False, + CWlists) if postStr: profileStr += postStr + separatorStr ctr += 1 @@ -109,9 +110,10 @@ def htmlFrontScreen(signingPrivateKeyPem: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - extraJson: {} = None, - pageNumber: int = None, - maxItemsPerPage: int = None) -> str: + extraJson: {}, + pageNumber: int, + maxItemsPerPage: int, + CWlists: {}) -> str: """Show the news instance front screen """ nickname = profileJson['preferredUsername'] @@ -182,7 +184,8 @@ def htmlFrontScreen(signingPrivateKeyPem: str, allowLocalNetworkAccess, theme, systemLanguage, maxLikeCount, - signingPrivateKeyPem) + licenseStr + signingPrivateKeyPem, + CWlists) + licenseStr # Footer which is only used for system accounts profileFooterStr = ' \n' diff --git a/webapp_moderation.py b/webapp_moderation.py index ef455e412..0a7298d05 100644 --- a/webapp_moderation.py +++ b/webapp_moderation.py @@ -52,7 +52,8 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the moderation feed as html This is what you see when selecting the "mod" timeline """ @@ -72,7 +73,7 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, - signingPrivateKeyPem) + signingPrivateKeyPem, CWlists) def htmlAccountInfo(cssCache: {}, translate: {}, diff --git a/webapp_post.py b/webapp_post.py index 4e5e6cbe6..ed67b537c 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -77,6 +77,7 @@ from webfinger import webfingerHandle from speaker import updateSpeaker from languages import autoTranslatePost from blocking import isBlocked +from blocking import addCWfromLists def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None: @@ -1205,7 +1206,8 @@ def individualPostAsHtml(signingPrivateKeyPem: str, manuallyApprovesFollowers: bool, showPublicOnly: bool, storeToCache: bool, - useCacheOnly: bool) -> str: + useCacheOnly: bool, + CWlists: {}) -> str: """ Shows a single post as html """ if not postJsonObject: @@ -1651,6 +1653,9 @@ def individualPostAsHtml(signingPrivateKeyPem: str, if newFooterStr: footerStr = newFooterStr + # add any content warning from the cwlists directory + addCWfromLists(postJsonObject, CWlists, translate) + postIsSensitive = False if postJsonObject['object'].get('sensitive'): # sensitive posts should have a summary @@ -1825,7 +1830,8 @@ def htmlIndividualPost(cssCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, signingPrivateKeyPem: str) -> str: + maxLikeCount: int, signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show an individual post as html """ postStr = '' @@ -1878,7 +1884,8 @@ def htmlIndividualPost(cssCache: {}, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - False, authorized, False, False, False, False) + False, authorized, False, False, False, False, + CWlists) messageId = removeIdEnding(postJsonObject['id']) # show the previous posts @@ -1910,7 +1917,8 @@ def htmlIndividualPost(cssCache: {}, themeName, systemLanguage, maxLikeCount, False, authorized, - False, False, False, False) + postStr + False, False, False, False, + CWlists) + postStr # show the following posts postFilename = locatePost(baseDir, nickname, domain, messageId) @@ -1944,7 +1952,8 @@ def htmlIndividualPost(cssCache: {}, themeName, systemLanguage, maxLikeCount, False, authorized, - False, False, False, False) + False, False, False, False, + CWlists) cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): cssFilename = baseDir + '/epicyon.css' @@ -1968,7 +1977,7 @@ def htmlPostReplies(cssCache: {}, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, CWlists: {}) -> str: """Show the replies to an individual post as html """ repliesStr = '' @@ -1991,7 +2000,8 @@ def htmlPostReplies(cssCache: {}, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - False, False, False, False, False, False) + False, False, False, False, False, False, + CWlists) cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): diff --git a/webapp_profile.py b/webapp_profile.py index 9de5871e4..2db29478b 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -89,7 +89,8 @@ def htmlProfileAfterSearch(cssCache: {}, accessKeys: {}, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show a profile page after a search for a fediverse address """ http = False @@ -306,7 +307,8 @@ def htmlProfileAfterSearch(cssCache: {}, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, - False, False, False, False, False, False) + False, False, False, False, False, False, + CWlists) i += 1 if i >= 8: break @@ -516,8 +518,9 @@ def htmlProfile(signingPrivateKeyPem: str, debug: bool, accessKeys: {}, city: str, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - extraJson: {} = None, pageNumber: int = None, - maxItemsPerPage: int = None) -> str: + extraJson: {}, pageNumber: int, + maxItemsPerPage: int, + CWlists: {}) -> str: """Show the profile page as html """ nickname = profileJson['preferredUsername'] @@ -539,8 +542,8 @@ def htmlProfile(signingPrivateKeyPem: str, newswire, theme, extraJson, allowLocalNetworkAccess, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, - pageNumber, maxItemsPerPage) + sharedItemsFederatedDomains, None, + pageNumber, maxItemsPerPage, CWlists) domain, port = getDomainFromActor(profileJson['id']) if not domain: @@ -897,7 +900,8 @@ def htmlProfile(signingPrivateKeyPem: str, allowLocalNetworkAccess, theme, systemLanguage, maxLikeCount, - signingPrivateKeyPem) + licenseStr + signingPrivateKeyPem, + CWlists) + licenseStr if not isGroup: if selected == 'following': profileStr += \ @@ -963,7 +967,8 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Shows posts on the profile screen These should only be public posts """ @@ -1008,7 +1013,8 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, themeName, systemLanguage, maxLikeCount, False, False, False, - True, False, False) + True, False, False, + CWlists) if postStr: profileStr += postStr + separatorStr ctr += 1 diff --git a/webapp_search.py b/webapp_search.py index c98f73c35..1f122b53d 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -598,7 +598,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, themeName: str, boxName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show a page containing search results for your post history """ if historysearch.startswith("'"): @@ -683,7 +684,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, themeName, systemLanguage, maxLikeCount, showIndividualPostIcons, showIndividualPostIcons, - False, False, False, False) + False, False, False, False, + CWlists) if postStr: historySearchForm += separatorStr + postStr index += 1 @@ -707,7 +709,8 @@ def htmlHashtagSearch(cssCache: {}, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show a page containing search results for a hashtag or after selecting a hashtag from the swarm """ @@ -865,7 +868,7 @@ def htmlHashtagSearch(cssCache: {}, showRepeats, showIcons, manuallyApprovesFollowers, showPublicOnly, - storeToCache, False) + storeToCache, False, CWlists) if postStr: hashtagSearchForm += separatorStr + postStr index += 1 diff --git a/webapp_timeline.py b/webapp_timeline.py index 91afad775..9f8c35f7d 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -446,7 +446,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the timeline as html """ enableTimingLog = False @@ -913,7 +914,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, boxName != 'dm', showIndividualPostIcons, manuallyApproveFollowers, - False, True, useCacheOnly) + False, True, useCacheOnly, + CWlists) _logTimelineTiming(enableTimingLog, timelineStartTime, boxName, '12') @@ -1137,7 +1139,8 @@ def htmlShares(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the shares timeline as html """ manuallyApproveFollowers = \ @@ -1162,7 +1165,8 @@ def htmlShares(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlWanted(cssCache: {}, defaultTimeline: str, @@ -1189,7 +1193,8 @@ def htmlWanted(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the wanted timeline as html """ manuallyApproveFollowers = \ @@ -1214,7 +1219,8 @@ def htmlWanted(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInbox(cssCache: {}, defaultTimeline: str, @@ -1242,7 +1248,8 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the inbox as html """ manuallyApproveFollowers = \ @@ -1267,7 +1274,8 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlBookmarks(cssCache: {}, defaultTimeline: str, @@ -1295,7 +1303,8 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the bookmarks as html """ manuallyApproveFollowers = \ @@ -1320,7 +1329,8 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxDMs(cssCache: {}, defaultTimeline: str, @@ -1348,7 +1358,8 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the DM timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1368,7 +1379,8 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxReplies(cssCache: {}, defaultTimeline: str, @@ -1396,7 +1408,8 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the replies timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1417,7 +1430,8 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxMedia(cssCache: {}, defaultTimeline: str, @@ -1445,7 +1459,8 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the media timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1466,7 +1481,8 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, @@ -1494,7 +1510,8 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the blogs timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1515,7 +1532,8 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, @@ -1544,7 +1562,8 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the features timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1565,7 +1584,8 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlInboxNews(cssCache: {}, defaultTimeline: str, @@ -1593,7 +1613,8 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the news timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1614,7 +1635,8 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) def htmlOutbox(cssCache: {}, defaultTimeline: str, @@ -1642,7 +1664,8 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, accessKeys: {}, systemLanguage: str, maxLikeCount: int, sharedItemsFederatedDomains: [], - signingPrivateKeyPem: str) -> str: + signingPrivateKeyPem: str, + CWlists: {}) -> str: """Show the Outbox as html """ manuallyApproveFollowers = \ @@ -1664,4 +1687,5 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, authorized, None, theme, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, - sharedItemsFederatedDomains, signingPrivateKeyPem) + sharedItemsFederatedDomains, signingPrivateKeyPem, + CWlists) From a6381f5c5050ace6e864ac427c79ed45c9adfd80 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 14:19:07 +0100 Subject: [PATCH 08/27] Tidying list --- cwlists/murdoch.json | 89 -------------------------------------------- 1 file changed, 89 deletions(-) diff --git a/cwlists/murdoch.json b/cwlists/murdoch.json index 52a4447ea..38932207d 100644 --- a/cwlists/murdoch.json +++ b/cwlists/murdoch.json @@ -8,162 +8,73 @@ "newscdn.com.au", "resourcesssl.newscdn.com.au", "thesun.co.uk", - "ads.thesun.co.uk", - "cdn.thesun.co.uk", - "cmp.cdn.thesun.co.uk", - "www.thesun.co.uk", "thetimes.co.uk", - "ads.thetimes.co.uk", - "www.thetimes.co.uk", "thesundaytimes.co.uk", "pressassociation.com", "news.co.uk", - "www.news.co.uk", "newscorpaustralia.com", - "www.newscorpaustralia.com", - "logflume.newscorpaustralia.com", "theaustralian.com.au", - "media.theaustralian.com.au", - "www.theaustralian.com.au", "aap.com.au", - "www.aap.com.au", "news.com.au", - "tags.news.com.au", - "go.news.com.au", - "origin.go.news.com.au", - "www.news.com.au", "skynews.com.au", - "www.skynews.com.au", - "www.skyweather.com.au", "skyweather.com.au", "australiannewschannel.com.au", "weeklytimesnow.com.au", - "www.weeklytimesnow.com.au", "dailytelegraph.com.au", - "www.dailytelegraph.com.au", "heraldsun.com.au", - "www.heraldsun.com.au", "geelongadvertiser.com.au", - "www.geelongadvertiser.com.au", "couriermail.com.au", - "www.couriermail.com.au", "thesundaymail.com.au", "goldcoastbulletin.com.au", - "www.goldcoastbulletin.com.au", "cairnspost.com.au", - "www.cairnspost.com.au", "townsvillebulletin.com.au", - "www.townsvillebulletin.com.au", "adelaidenow.com.au", - "www.adelaidenow.com.au", "themercury.com.au", - "www.themercury.com.au", "ntnews.com.au", - "www.ntnews.com.au", "postcourier.com.pg", "nypost.com", - "embeds.nypost.com", "pagesix.com", "realtor.com", "wsj.com", - "www.wsj.com", - "accounts.wsj.com", - "video-api.wsj.com", "wsj.net", - "images.wsj.net", - "m4.wsj.net", - "m.wsj.net", - "s.wsj.net", - "si.wsj.net", - "sts3.wsj.net", - "vir.wsj.net", "foxnews.com", - "www.foxnews.com", - "a57.foxnews.com", - "static.foxnews.com", "fncstatic.com", - "global.fncstatic.com", "foxnewsgo.com", - "www.foxbusiness.com", "fox.com", - "strike.fox.com", "foxbusiness.com", - "www.foxbusiness.com", "foxsports.com", "fssta.com", - "b.fssta.com", "foxsports.com.au", "dowjones.com", - "www.dowjones.com", - "images.dowjones.com", "factiva.com", "barrons.com", - "accounts.barrons.com", - "asset.barrons.com", - "images.barrons.com", - "video-api.barrons.com", - "www.barrons.com", "marketwatch.com", - "www.marketwatch.com", "heatst.com", "fnlondon.com", - "www.fnlondon.com", - "assets.fnlondon.com", - "images.fnlondon.com", "mansionglobal.com", - "images.mansionglobal.com", - "www.mansionglobal.com", "spindices.com", "spglobal.com", - "www.spglobal.com", "talksport.com", - "ads.talksport.com", - "cdn259.talksport.com", "harpercollins.com", - "www.harpercollins.com", "bestrecipes.com.au", - "img.bestrecipes.com.au", - "www.bestrecipes.com.au", "hipages.com.au", - "img.hipages.com.au", "homeimprovementpages.com.au", - "assets.homeimprovementpages.com.au", "odds.com.au", - "www.odds.com.au", "onebigswitch.com.au", "suddenly.com.au", - "www.suddenly.com.au", "supercoach.com.au", "punters.com.au", - "www.punters.com.au", "kayosports.com.au", "foxtel.com.au", - "www.foxtel.com.au", "newscorp.com", "storyful.com", "vogue.com.au", - "media.vogue.com.au", - "www.vogue.com.au", "taste.com.au", - "img.taste.com.au", - "www.taste.com.au", "kidspot.com.au", - "go.kidspot.com.au", - "origin.go.kidspot.com.au", - "img.kidspot.com.au", - "www.kidspot.com.au", "bodyandsoul.com.au", - "go.bodyandsoul.com.au", - "origin.bodyandsoul.com.au", - "www.bodyandsoul.com.au", "realcommercial.com.au", - "www.realcommercial.com.au", "reastatic.net", - "au.reastatic.net", - "rui.au.reastatic.net", - "s1.rui.au.reastatic.net", "realestate.com.au", - "www.realestate.com.au", "whereilive.com.au", "www.newsoftheworld.co.uk", "newsoftheworld.co.uk" From b91c6282c766e3ba3d8dc8e56bc6216b39350cde Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 14:36:40 +0100 Subject: [PATCH 09/27] Configuration parameter for enabling cw lists --- blocking.py | 4 +++- daemon.py | 3 ++- tests.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/blocking.py b/blocking.py index f97f52549..148191c52 100644 --- a/blocking.py +++ b/blocking.py @@ -876,7 +876,7 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool: return False -def loadCWLists(baseDir: str, verbose: bool) -> {}: +def loadCWLists(baseDir: str, verbose: bool, listsEnabled: str) -> {}: """Load lists used for content warnings """ if not os.path.isdir(baseDir + '/cwlists'): @@ -893,6 +893,8 @@ def loadCWLists(baseDir: str, verbose: bool) -> {}: continue if not listJson.get('name'): continue + if listJson['name'] not in listsEnabled: + continue if not listJson.get('words') and not listJson.get('domains'): continue name = listJson['name'] diff --git a/daemon.py b/daemon.py index d5e2af85e..afff8991f 100644 --- a/daemon.py +++ b/daemon.py @@ -17199,7 +17199,8 @@ def runDaemon(defaultReplyIntervalHours: int, updateBlockedCache(baseDir, httpd.blockedCache, httpd.blockedCacheLastUpdated, httpd.blockedCacheUpdateSecs) - httpd.CWlists = loadCWLists(baseDir, True) + httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") + httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) # cache to store css files httpd.cssCache = {} diff --git a/tests.py b/tests.py index addec4342..6a0dfdada 100644 --- a/tests.py +++ b/tests.py @@ -5769,7 +5769,7 @@ def _testWordsSimilarity() -> None: def _testAddCWfromLists(baseDir: str) -> None: print('testAddCWfromLists') translate = {} - CWlists = loadCWLists(baseDir, True) + CWlists = loadCWLists(baseDir, True, "Murdoch press") assert CWlists postJsonObject = { From 142a68a97ceaf822dcc8c0560b0fb3f4cf2f24b3 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 14:43:48 +0100 Subject: [PATCH 10/27] Default cw lists --- daemon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index afff8991f..099a04901 100644 --- a/daemon.py +++ b/daemon.py @@ -17199,8 +17199,6 @@ def runDaemon(defaultReplyIntervalHours: int, updateBlockedCache(baseDir, httpd.blockedCache, httpd.blockedCacheLastUpdated, httpd.blockedCacheUpdateSecs) - httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") - httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) # cache to store css files httpd.cssCache = {} @@ -17219,6 +17217,10 @@ def runDaemon(defaultReplyIntervalHours: int, if not os.path.isdir(baseDir + '/accounts/news@' + domain): print('Creating news inbox: news@' + domain) createNewsInbox(baseDir, domain, port, httpPrefix) + setConfigParam(baseDir, "listsEnabled", "Murdoch press") + + httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") + httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) # set the avatar for the news account httpd.themeName = getConfigParam(baseDir, 'theme') From 0c0cbf94859b305fa328d90f663a680e3843a35b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 15:06:25 +0100 Subject: [PATCH 11/27] Enabled lists set from commandline --- daemon.py | 8 ++++++-- epicyon.py | 14 +++++++++++++- tests.py | 12 ++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/daemon.py b/daemon.py index 099a04901..3323b2c4a 100644 --- a/daemon.py +++ b/daemon.py @@ -16864,7 +16864,8 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: break -def runDaemon(defaultReplyIntervalHours: int, +def runDaemon(listsEnabled: str, + defaultReplyIntervalHours: int, lowBandwidth: bool, maxLikeCount: int, sharedItemsFederatedDomains: [], @@ -17219,7 +17220,10 @@ def runDaemon(defaultReplyIntervalHours: int, createNewsInbox(baseDir, domain, port, httpPrefix) setConfigParam(baseDir, "listsEnabled", "Murdoch press") - httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") + if listsEnabled: + httpd.listsEnabled = listsEnabled + else: + httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) # set the avatar for the news account diff --git a/epicyon.py b/epicyon.py index 897ddafa3..e9f39e100 100644 --- a/epicyon.py +++ b/epicyon.py @@ -112,6 +112,10 @@ def str2bool(v) -> bool: parser = argparse.ArgumentParser(description='ActivityPub Server') +parser.add_argument('--listsEnabled', type=str, + default=None, + help='Names of content warning lists enabled. ' + + 'See the cwlists directory') parser.add_argument('--userAgentBlocks', type=str, default=None, help='List of blocked user agents, separated by commas') @@ -3011,6 +3015,13 @@ if userAgentsBlockedStr: for agentBlockStr in agentBlocksList: userAgentsBlocked.append(agentBlockStr.strip()) +listsEnabled = '' +if args.listsEnabled: + listsEnabled = args.listsEnabled + setConfigParam(baseDir, 'listsEnabled', listsEnabled) +else: + listsEnabled = getConfigParam(baseDir, 'listsEnabled') + city = \ getConfigParam(baseDir, 'city') if city is not None: @@ -3064,7 +3075,8 @@ if args.defaultCurrency: print('Default currency set to ' + args.defaultCurrency) if __name__ == "__main__": - runDaemon(args.defaultReplyIntervalHours, + runDaemon(listsEnabled, + args.defaultReplyIntervalHours, args.lowBandwidth, args.maxLikeCount, sharedItemsFederatedDomains, userAgentsBlocked, diff --git a/tests.py b/tests.py index 6a0dfdada..43366cb4a 100644 --- a/tests.py +++ b/tests.py @@ -817,8 +817,9 @@ def createServerAlice(path: str, domain: str, port: int, userAgentsBlocked = [] maxLikeCount = 10 defaultReplyIntervalHours = 9999999999 + listsEnabled = '' print('Server running: Alice') - runDaemon(defaultReplyIntervalHours, + runDaemon(listsEnabled, defaultReplyIntervalHours, lowBandwidth, maxLikeCount, sharedItemsFederatedDomains, userAgentsBlocked, @@ -955,8 +956,9 @@ def createServerBob(path: str, domain: str, port: int, userAgentsBlocked = [] maxLikeCount = 10 defaultReplyIntervalHours = 9999999999 + listsEnabled = '' print('Server running: Bob') - runDaemon(defaultReplyIntervalHours, + runDaemon(listsEnabled, defaultReplyIntervalHours, lowBandwidth, maxLikeCount, sharedItemsFederatedDomains, userAgentsBlocked, @@ -1022,8 +1024,9 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], maxLikeCount = 10 lowBandwidth = True defaultReplyIntervalHours = 9999999999 + listsEnabled = '' print('Server running: Eve') - runDaemon(defaultReplyIntervalHours, + runDaemon(listsEnabled, defaultReplyIntervalHours, lowBandwidth, maxLikeCount, sharedItemsFederatedDomains, userAgentsBlocked, @@ -1091,8 +1094,9 @@ def createServerGroup(path: str, domain: str, port: int, maxLikeCount = 10 lowBandwidth = True defaultReplyIntervalHours = 9999999999 + listsEnabled = '' print('Server running: Group') - runDaemon(defaultReplyIntervalHours, + runDaemon(listsEnabled, defaultReplyIntervalHours, lowBandwidth, maxLikeCount, sharedItemsFederatedDomains, userAgentsBlocked, From 6a4cb89f58d39d044c87263a3f6bb24f965eb467 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 15:41:05 +0100 Subject: [PATCH 12/27] Check that there are some lists enabled --- blocking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blocking.py b/blocking.py index 148191c52..b55282d55 100644 --- a/blocking.py +++ b/blocking.py @@ -879,6 +879,8 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool: def loadCWLists(baseDir: str, verbose: bool, listsEnabled: str) -> {}: """Load lists used for content warnings """ + if not listsEnabled: + return {} if not os.path.isdir(baseDir + '/cwlists'): return {} result = {} From 6a79a6d3c774d2474600f542dbe05f024ad151d6 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 17:21:31 +0100 Subject: [PATCH 13/27] Another list --- cwlists/ukrwthinktanks.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 cwlists/ukrwthinktanks.json diff --git a/cwlists/ukrwthinktanks.json b/cwlists/ukrwthinktanks.json new file mode 100644 index 000000000..b674ff49b --- /dev/null +++ b/cwlists/ukrwthinktanks.json @@ -0,0 +1,24 @@ +{ + "name": "UK RW Think Tanks", + "warning": "UK RW Think Tank", + "words": [ + "Adam Smith Institute", + "Bow Group", + "Centre for Policy Studies", + "Centre for Social Justice", + "Chatham House", + "Institute of Economic Affairs", + "Legatum Institute", + "Policy Exchange" + ], + "domains": [ + "adamsmith.org", + "bowgroup.org", + "cps.org.uk", + "centreforsocialjustice.org.uk", + "chathamhouse.org", + "iea.org.uk", + "https://li.com", + "policyexchange.org.uk" + ] +} From 48c298f5f8920c3b77634ca9ec81ff9083a8cc21 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 17:58:01 +0100 Subject: [PATCH 14/27] Show selectable lists --- daemon.py | 4 +++- translations/ar.json | 3 ++- translations/ca.json | 3 ++- translations/cy.json | 3 ++- translations/de.json | 3 ++- translations/en.json | 3 ++- translations/es.json | 3 ++- translations/fr.json | 3 ++- translations/ga.json | 3 ++- translations/hi.json | 3 ++- translations/it.json | 3 ++- translations/ja.json | 3 ++- translations/ku.json | 3 ++- translations/oc.json | 3 ++- translations/pt.json | 3 ++- translations/ru.json | 3 ++- translations/sw.json | 3 ++- translations/zh.json | 3 ++- webapp_profile.py | 24 +++++++++++++++++++++--- 19 files changed, 58 insertions(+), 21 deletions(-) diff --git a/daemon.py b/daemon.py index 3323b2c4a..e2f63938a 100644 --- a/daemon.py +++ b/daemon.py @@ -11774,7 +11774,9 @@ class PubServer(BaseHTTPRequestHandler): city, self.server.userAgentsBlocked, accessKeys, - defaultReplyIntervalHours).encode('utf-8') + defaultReplyIntervalHours, + self.server.CWlists, + self.server.listsEnabled).encode('utf-8') if msg: msglen = len(msg) self._set_headers('text/html', msglen, diff --git a/translations/ar.json b/translations/ar.json index 325f09dc4..2ae899465 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -488,5 +488,6 @@ "Members": "أعضاء", "Join": "انضم", "Leave": "يترك", - "System Monitor": "مراقب النظام" + "System Monitor": "مراقب النظام", + "Add content warnings for the following sites": "أضف تحذيرات المحتوى للمواقع التالية" } diff --git a/translations/ca.json b/translations/ca.json index 8e7b37c08..f4861d724 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -488,5 +488,6 @@ "Members": "Membres", "Join": "Uneix-te", "Leave": "Marxa", - "System Monitor": "Monitor del sistema" + "System Monitor": "Monitor del sistema", + "Add content warnings for the following sites": "Afegiu advertiments de contingut per als llocs següents" } diff --git a/translations/cy.json b/translations/cy.json index 97daf5510..f25c5a27c 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -488,5 +488,6 @@ "Members": "Aelodau", "Join": "Ymunwch", "Leave": "Gadewch", - "System Monitor": "Monitor System" + "System Monitor": "Monitor System", + "Add content warnings for the following sites": "Ychwanegwch rybuddion cynnwys ar gyfer y gwefannau canlynol" } diff --git a/translations/de.json b/translations/de.json index 0667373c2..e2760a864 100644 --- a/translations/de.json +++ b/translations/de.json @@ -488,5 +488,6 @@ "Members": "Mitglieder", "Join": "Verbinden", "Leave": "Verlassen", - "System Monitor": "Systemmonitor" + "System Monitor": "Systemmonitor", + "Add content warnings for the following sites": "Inhaltswarnungen für die folgenden Websites hinzufügen" } diff --git a/translations/en.json b/translations/en.json index d7b71a414..4e8d3ec13 100644 --- a/translations/en.json +++ b/translations/en.json @@ -488,5 +488,6 @@ "Members": "Members", "Join": "Join", "Leave": "Leave", - "System Monitor": "System Monitor" + "System Monitor": "System Monitor", + "Add content warnings for the following sites": "Add content warnings for the following sites" } diff --git a/translations/es.json b/translations/es.json index 1bd72c84e..67cf33835 100644 --- a/translations/es.json +++ b/translations/es.json @@ -488,5 +488,6 @@ "Members": "Miembros", "Join": "Entrar", "Leave": "Dejar", - "System Monitor": "Monitor del sistema" + "System Monitor": "Monitor del sistema", + "Add content warnings for the following sites": "Agregue advertencias de contenido para los siguientes sitios" } diff --git a/translations/fr.json b/translations/fr.json index 16f396e53..056a914c0 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -488,5 +488,6 @@ "Members": "Membres", "Join": "Rejoindre", "Leave": "Laisser", - "System Monitor": "Moniteur système" + "System Monitor": "Moniteur système", + "Add content warnings for the following sites": "Ajouter des avertissements de contenu pour les sites suivants" } diff --git a/translations/ga.json b/translations/ga.json index eb0b12af3..73f740bcf 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -488,5 +488,6 @@ "Members": "Baill", "Join": "Bí páirteach", "Leave": "Fág", - "System Monitor": "Monatóir Córais" + "System Monitor": "Monatóir Córais", + "Add content warnings for the following sites": "Cuir rabhaidh ábhair leis na suíomhanna seo a leanas" } diff --git a/translations/hi.json b/translations/hi.json index b93a3180d..c4e475f2d 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -488,5 +488,6 @@ "Members": "सदस्यों", "Join": "शामिल हों", "Leave": "छोड़ना", - "System Monitor": "सिस्टम मॉनिटर" + "System Monitor": "सिस्टम मॉनिटर", + "Add content warnings for the following sites": "निम्नलिखित साइटों के लिए सामग्री चेतावनियाँ जोड़ें" } diff --git a/translations/it.json b/translations/it.json index 4ea931172..4648aedae 100644 --- a/translations/it.json +++ b/translations/it.json @@ -488,5 +488,6 @@ "Members": "Membri", "Join": "Aderire", "Leave": "Lasciare", - "System Monitor": "Monitor di sistema" + "System Monitor": "Monitor di sistema", + "Add content warnings for the following sites": "Aggiungi avvisi sui contenuti per i seguenti siti" } diff --git a/translations/ja.json b/translations/ja.json index d751c83aa..0fe46e4f4 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -488,5 +488,6 @@ "Members": "メンバー", "Join": "加入", "Leave": "離れる", - "System Monitor": "システムモニター" + "System Monitor": "システムモニター", + "Add content warnings for the following sites": "次のサイトのコンテンツ警告を追加します" } diff --git a/translations/ku.json b/translations/ku.json index bf383bdfe..1c4f7dd19 100644 --- a/translations/ku.json +++ b/translations/ku.json @@ -488,5 +488,6 @@ "Members": "Endam", "Join": "Bihevgirêdan", "Leave": "Terikandin", - "System Monitor": "System Monitor" + "System Monitor": "System Monitor", + "Add content warnings for the following sites": "Ji bo malperên jêrîn hişyariyên naverokê zêde bikin" } diff --git a/translations/oc.json b/translations/oc.json index c2e6f0c8f..d898dfa43 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -484,5 +484,6 @@ "Members": "Members", "Join": "Join", "Leave": "Leave", - "System Monitor": "System Monitor" + "System Monitor": "System Monitor", + "Add content warnings for the following sites": "Add content warnings for the following sites" } diff --git a/translations/pt.json b/translations/pt.json index 75444a1b5..789df52ea 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -488,5 +488,6 @@ "Members": "Membros", "Join": "Juntar", "Leave": "Sair", - "System Monitor": "Monitor de Sistema" + "System Monitor": "Monitor de Sistema", + "Add content warnings for the following sites": "Adicione avisos de conteúdo para os seguintes sites" } diff --git a/translations/ru.json b/translations/ru.json index 83597b5b0..2352871d7 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -488,5 +488,6 @@ "Members": "Члены", "Join": "Присоединиться", "Leave": "Оставлять", - "System Monitor": "Системный монитор" + "System Monitor": "Системный монитор", + "Add content warnings for the following sites": "Добавить предупреждения о содержании для следующих сайтов" } diff --git a/translations/sw.json b/translations/sw.json index c658897b2..3e7aa7b8d 100644 --- a/translations/sw.json +++ b/translations/sw.json @@ -488,5 +488,6 @@ "Members": "Wanachama", "Join": "Jiunge", "Leave": "Ondoka", - "System Monitor": "Ufuatiliaji wa Mfumo" + "System Monitor": "Ufuatiliaji wa Mfumo", + "Add content warnings for the following sites": "Ongeza maonyo ya yaliyomo kwa wavuti zifuatazo" } diff --git a/translations/zh.json b/translations/zh.json index db16066e8..e7104146b 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -488,5 +488,6 @@ "Members": "会员", "Join": "加入", "Leave": "离开", - "System Monitor": "系统监视器" + "System Monitor": "系统监视器", + "Add content warnings for the following sites": "为以下网站添加内容警告" } diff --git a/webapp_profile.py b/webapp_profile.py index 2db29478b..4cf45c2ef 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1518,7 +1518,8 @@ def _htmlEditProfileSharedItems(baseDir: str, nickname: str, domain: str, def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, userAgentsBlocked: str, translate: {}, - replyIntervalHours: int) -> str: + replyIntervalHours: int, + CWlists: {}, listsEnabled: str) -> str: """Filtering and blocking section of edit profile screen """ filterStr = '' @@ -1674,6 +1675,21 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 'userAgentsBlockedStr', userAgentsBlockedStr, 200, '', False) + idx = 'Add content warnings for the following sites' + editProfileForm += \ + '\n' + + for listName, item in CWlists.items(): + name = listName + variableName = 'list' + name.replace(' ', '').replace("'", '') + if listName in listsEnabled: + listIsEnabled = True + else: + listIsEnabled = False + if translate.get(name): + name = translate[name] + editProfileForm += editCheckBox(name, variableName, listIsEnabled) + editProfileForm += endEditSection() return editProfileForm @@ -1956,7 +1972,8 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, textModeBanner: str, city: str, userAgentsBlocked: str, accessKeys: {}, - defaultReplyIntervalHours: int) -> str: + defaultReplyIntervalHours: int, + CWlists: {}, listsEnabled: str) -> str: """Shows the edit profile screen """ path = path.replace('/inbox', '').replace('/outbox', '') @@ -2152,7 +2169,8 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, editProfileForm += \ _htmlEditProfileFiltering(baseDir, nickname, domain, userAgentsBlocked, translate, - replyIntervalHours) + replyIntervalHours, + CWlists, listsEnabled) # git projects section editProfileForm += \ From 8de2a2423582748fbe10346d7fb0cadf2bee1d3f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 18:12:10 +0100 Subject: [PATCH 15/27] Check that lists exist before adding header --- webapp_profile.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 4cf45c2ef..15320c01c 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1675,10 +1675,7 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 'userAgentsBlockedStr', userAgentsBlockedStr, 200, '', False) - idx = 'Add content warnings for the following sites' - editProfileForm += \ - '\n' - + CWlistsStr = '' for listName, item in CWlists.items(): name = listName variableName = 'list' + name.replace(' ', '').replace("'", '') @@ -1688,7 +1685,12 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, listIsEnabled = False if translate.get(name): name = translate[name] - editProfileForm += editCheckBox(name, variableName, listIsEnabled) + CWlistsStr += editCheckBox(name, variableName, listIsEnabled) + if CWlistsStr: + idx = 'Add content warnings for the following sites' + editProfileForm += \ + '\n' + \ + CWlistsStr editProfileForm += endEditSection() return editProfileForm From 63e68c7af3cf3b1b6e465bb05587a51804bfba79 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 18:14:05 +0100 Subject: [PATCH 16/27] Use name field --- webapp_profile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index 15320c01c..aab8eccb5 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1677,7 +1677,9 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, CWlistsStr = '' for listName, item in CWlists.items(): - name = listName + if not item.get('name'): + continue + name = item['name'] variableName = 'list' + name.replace(' ', '').replace("'", '') if listName in listsEnabled: listIsEnabled = True From 7c2da78c553c6e9363569c189bd4c9a58265ad09 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 18:27:27 +0100 Subject: [PATCH 17/27] Debug --- webapp_profile.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index aab8eccb5..dccb6036f 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1676,10 +1676,8 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 200, '', False) CWlistsStr = '' - for listName, item in CWlists.items(): - if not item.get('name'): - continue - name = item['name'] + for name, item in CWlists.items(): + print('CWlistsStr: ' + name) variableName = 'list' + name.replace(' ', '').replace("'", '') if listName in listsEnabled: listIsEnabled = True From a5552519506e0398cacca33f070a74317b219c7f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 18:31:12 +0100 Subject: [PATCH 18/27] Debug --- webapp_profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index dccb6036f..8b2877cf8 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1676,10 +1676,11 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 200, '', False) CWlistsStr = '' + print('CWlistsStr 0: ' + str(CWlists)) for name, item in CWlists.items(): print('CWlistsStr: ' + name) variableName = 'list' + name.replace(' ', '').replace("'", '') - if listName in listsEnabled: + if name in listsEnabled: listIsEnabled = True else: listIsEnabled = False From 7d2b5aa1b35e24767eb71692865a86cef7b798d7 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 18:36:08 +0100 Subject: [PATCH 19/27] Debug --- daemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon.py b/daemon.py index e2f63938a..531240c32 100644 --- a/daemon.py +++ b/daemon.py @@ -17227,6 +17227,7 @@ def runDaemon(listsEnabled: str, else: httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) + print('CWlistsStr httpd.CWlists: ' + str(httpd.CWlists)) # set the avatar for the news account httpd.themeName = getConfigParam(baseDir, 'theme') From 9ecb745d3dfff190cfce3ba698b062afbd675d5a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:00:25 +0100 Subject: [PATCH 20/27] Load all lists and then enable them selectively --- blocking.py | 13 ++--- daemon.py | 108 +++++++++++++++++++++++++++--------------- inbox.py | 52 ++++++++++++-------- outbox.py | 5 +- schedule.py | 3 +- tests.py | 8 ++-- webapp_confirm.py | 4 +- webapp_frontscreen.py | 9 ++-- webapp_moderation.py | 4 +- webapp_post.py | 18 +++---- webapp_profile.py | 15 +++--- webapp_search.py | 10 ++-- webapp_timeline.py | 48 +++++++++---------- 13 files changed, 176 insertions(+), 121 deletions(-) diff --git a/blocking.py b/blocking.py index b55282d55..0ba2d3623 100644 --- a/blocking.py +++ b/blocking.py @@ -876,11 +876,9 @@ def brochModeLapses(baseDir: str, lapseDays: int = 7) -> bool: return False -def loadCWLists(baseDir: str, verbose: bool, listsEnabled: str) -> {}: +def loadCWLists(baseDir: str, verbose: bool) -> {}: """Load lists used for content warnings """ - if not listsEnabled: - return {} if not os.path.isdir(baseDir + '/cwlists'): return {} result = {} @@ -895,8 +893,6 @@ def loadCWLists(baseDir: str, verbose: bool, listsEnabled: str) -> {}: continue if not listJson.get('name'): continue - if listJson['name'] not in listsEnabled: - continue if not listJson.get('words') and not listJson.get('domains'): continue name = listJson['name'] @@ -906,10 +902,13 @@ def loadCWLists(baseDir: str, verbose: bool, listsEnabled: str) -> {}: return result -def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}) -> None: +def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}, + listsEnabled: str) -> None: """Adds content warnings by matching the post content against domains or keywords """ + if not listsEnabled: + return if not postJsonObject['object'].get('content'): return cw = '' @@ -918,6 +917,8 @@ def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}) -> None: content = postJsonObject['object']['content'] for name, item in CWlists.items(): + if name not in listsEnabled: + continue if not item.get('warning'): continue warning = item['warning'] diff --git a/daemon.py b/daemon.py index 531240c32..c24451bcb 100644 --- a/daemon.py +++ b/daemon.py @@ -1231,7 +1231,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.maxLikeCount, self.server.maxRecentPosts, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) def _postToOutboxThread(self, messageJson: {}) -> bool: """Creates a thread to send a post @@ -3010,7 +3011,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -3069,7 +3071,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if historyStr: msg = historyStr.encode('utf-8') msglen = len(msg) @@ -3108,7 +3111,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if bookmarksStr: msg = bookmarksStr.encode('utf-8') msglen = len(msg) @@ -3213,7 +3217,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if profileStr: msg = profileStr.encode('utf-8') msglen = len(msg) @@ -6734,7 +6739,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -6957,7 +6963,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) self.server.GETbusy = False actorAbsolute = self._getInstanceUrl(callingDomain) + actor @@ -7439,7 +7446,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Liked post not found: ' + likedPostFilename) # clear the icon from the cache so that it gets updated @@ -7600,7 +7608,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Unliked post not found: ' + likedPostFilename) # clear the icon from the cache so that it gets updated @@ -7736,7 +7745,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Bookmarked post not found: ' + bookmarkFilename) # self._postToOutbox(bookmarkJson, self.server.projectVersion, None) @@ -7871,7 +7881,8 @@ class PubServer(BaseHTTPRequestHandler): showIndividualPostIcons, manuallyApproveFollowers, False, True, False, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Unbookmarked post not found: ' + bookmarkFilename) self.server.GETbusy = False @@ -7974,7 +7985,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if deleteStr: deleteStrLen = len(deleteStr) self._set_headers('text/html', deleteStrLen, @@ -8083,7 +8095,8 @@ class PubServer(BaseHTTPRequestHandler): manuallyApproveFollowers, showPublicOnly, storeToCache, useCacheOnly, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Muted post not found: ' + muteFilename) @@ -8193,7 +8206,8 @@ class PubServer(BaseHTTPRequestHandler): manuallyApproveFollowers, showPublicOnly, storeToCache, useCacheOnly, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) else: print('WARN: Unmuted post not found: ' + muteFilename) self.server.GETbusy = False @@ -8317,7 +8331,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8417,7 +8432,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8526,7 +8542,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, rolesList, - None, None, self.server.CWlists) + None, None, self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8640,7 +8657,8 @@ class PubServer(BaseHTTPRequestHandler): sharedItemsFederatedDomains, skills, None, None, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8790,7 +8808,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.maxLikeCount, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9037,7 +9056,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) if GETstartTime: fitnessPerformance(GETstartTime, self.server.fitness, @@ -9186,7 +9206,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9328,7 +9349,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9469,7 +9491,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9610,7 +9633,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9760,7 +9784,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -9908,7 +9933,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10015,7 +10041,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10100,7 +10127,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10222,7 +10250,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10361,7 +10390,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10491,7 +10521,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, sharedItemsFederatedDomains, self.server.signingPrivateKeyPem, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10619,7 +10650,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.sharedItemsFederatedDomains, shares, pageNumber, sharesPerPage, - self.server.CWlists) + self.server.CWlists, + self.server.listsEnabled) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -10744,7 +10776,8 @@ class PubServer(BaseHTTPRequestHandler): following, pageNumber, followsPerPage, - self.server.CWlists).encode('utf-8') + self.server.CWlists, + self.server.listsEnabled).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -10868,7 +10901,8 @@ class PubServer(BaseHTTPRequestHandler): followers, pageNumber, followsPerPage, - self.server.CWlists).encode('utf-8') + self.server.CWlists, + self.server.listsEnabled).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -11008,7 +11042,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.maxLikeCount, self.server.sharedItemsFederatedDomains, None, None, None, - self.server.CWlists).encode('utf-8') + self.server.CWlists, + self.server.listsEnabled).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain, False) @@ -17226,8 +17261,7 @@ def runDaemon(listsEnabled: str, httpd.listsEnabled = listsEnabled else: httpd.listsEnabled = getConfigParam(baseDir, "listsEnabled") - httpd.CWlists = loadCWLists(baseDir, True, httpd.listsEnabled) - print('CWlistsStr httpd.CWlists: ' + str(httpd.CWlists)) + httpd.CWlists = loadCWLists(baseDir, True) # set the avatar for the news account httpd.themeName = getConfigParam(baseDir, 'theme') diff --git a/inbox.py b/inbox.py index 4b1fd21c4..8df064200 100644 --- a/inbox.py +++ b/inbox.py @@ -275,7 +275,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> None: + CWlists: {}, + listsEnabled: str) -> None: """Converts the json post into html and stores it in a cache This enables the post to be quickly displayed later """ @@ -300,7 +301,7 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, notDM, True, True, False, True, False, - CWlists) + CWlists, listsEnabled) def validInbox(baseDir: str, nickname: str, domain: str) -> bool: @@ -982,7 +983,8 @@ def _receiveLike(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, CWlists: {}) -> bool: + maxLikeCount: int, CWlists: {}, + listsEnabled: str) -> bool: """Receives a Like activity within the POST section of HTTPServer """ if messageJson['type'] != 'Like': @@ -1084,7 +1086,8 @@ def _receiveLike(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False, CWlists) + False, True, False, CWlists, + listsEnabled) return True @@ -1102,7 +1105,8 @@ def _receiveUndoLike(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, CWlists: {}) -> bool: + maxLikeCount: int, CWlists: {}, + listsEnabled: str) -> bool: """Receives an undo like activity within the POST section of HTTPServer """ if messageJson['type'] != 'Undo': @@ -1193,7 +1197,8 @@ def _receiveUndoLike(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False, CWlists) + False, True, False, CWlists, + listsEnabled) return True @@ -1210,7 +1215,8 @@ def _receiveBookmark(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, CWlists: {}) -> bool: + maxLikeCount: int, CWlists: {}, + listsEnabled: {}) -> bool: """Receives a bookmark activity within the POST section of HTTPServer """ if not messageJson.get('type'): @@ -1301,7 +1307,8 @@ def _receiveBookmark(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False, CWlists) + False, True, False, CWlists, + listsEnabled) return True @@ -1318,7 +1325,8 @@ def _receiveUndoBookmark(recentPostsCache: {}, peertubeInstances: [], allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, - maxLikeCount: int, CWlists: {}) -> bool: + maxLikeCount: int, CWlists: {}, + listsEnabled: str) -> bool: """Receives an undo bookmark activity within the POST section of HTTPServer """ if not messageJson.get('type'): @@ -1410,7 +1418,7 @@ def _receiveUndoBookmark(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False, CWlists) + False, True, False, CWlists, listsEnabled) return True @@ -1504,7 +1512,8 @@ def _receiveAnnounce(recentPostsCache: {}, maxRecentPosts: int, allowDeletion: bool, peertubeInstances: [], - maxLikeCount: int, CWlists: {}) -> bool: + maxLikeCount: int, CWlists: {}, + listsEnabled: str) -> bool: """Receives an announce activity within the POST section of HTTPServer """ if messageJson['type'] != 'Announce': @@ -1616,7 +1625,8 @@ def _receiveAnnounce(recentPostsCache: {}, maxLikeCount, notDM, showIndividualPostIcons, manuallyApproveFollowers, - False, True, False, CWlists) + False, True, False, CWlists, + listsEnabled) if not announceHtml: print('WARN: Unable to generate html for announce ' + str(messageJson)) @@ -2554,7 +2564,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, maxLikeCount: int, signingPrivateKeyPem: str, defaultReplyIntervalHours: int, - CWlists: {}) -> bool: + CWlists: {}, listsEnabled: str) -> bool: """ Anything which needs to be done after initial checks have passed """ actor = keyId @@ -2584,7 +2594,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount, CWlists): + maxLikeCount, CWlists, listsEnabled): if debug: print('DEBUG: Like accepted from ' + actor) return False @@ -2606,7 +2616,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount, CWlists): + maxLikeCount, CWlists, listsEnabled): if debug: print('DEBUG: Undo like accepted from ' + actor) return False @@ -2628,7 +2638,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount, CWlists): + maxLikeCount, CWlists, listsEnabled): if debug: print('DEBUG: Bookmark accepted from ' + actor) return False @@ -2650,7 +2660,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, - maxLikeCount, CWlists): + maxLikeCount, CWlists, listsEnabled): if debug: print('DEBUG: Undo bookmark accepted from ' + actor) return False @@ -2676,7 +2686,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, maxRecentPosts, allowDeletion, peertubeInstances, - maxLikeCount, CWlists): + maxLikeCount, CWlists, listsEnabled): if debug: print('DEBUG: Announce accepted from ' + actor) @@ -2958,7 +2968,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, themeName, systemLanguage, maxLikeCount, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) if debug: timeDiff = \ str(int((time.time() - htmlCacheStartTime) * @@ -3631,6 +3641,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if not os.path.isfile(sharedInboxPostFilename): saveJson(queueJson['post'], sharedInboxPostFilename) + listsEnabled = getConfigParam(baseDir, "listsEnabled") + # for posts addressed to specific accounts for handle, capsId in recipientsDict.items(): destination = \ @@ -3662,7 +3674,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, maxLikeCount, signingPrivateKeyPem, defaultReplyIntervalHours, - CWlists) + CWlists, listsEnabled) if debug: pprint(queueJson['post']) print('Queue: Queue post accepted') diff --git a/outbox.py b/outbox.py index 4db77d0cd..522f7a402 100644 --- a/outbox.py +++ b/outbox.py @@ -196,7 +196,8 @@ def postMessageToOutbox(session, translate: {}, signingPrivateKeyPem: str, peertubeInstances: str, theme: str, maxLikeCount: int, - maxRecentPosts: int, CWlists: {}) -> bool: + maxRecentPosts: int, CWlists: {}, + listsEnabled: str) -> bool: """post is received by the outbox Client to server message post https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery @@ -468,7 +469,7 @@ def postMessageToOutbox(session, translate: {}, showIndividualPostIcons, manuallyApproveFollowers, False, True, useCacheOnly, - CWlists) + CWlists, listsEnabled) if outboxAnnounce(recentPostsCache, baseDir, messageJson, debug): diff --git a/schedule.py b/schedule.py index c7a508d4a..09212396c 100644 --- a/schedule.py +++ b/schedule.py @@ -125,7 +125,8 @@ def _updatePostSchedule(baseDir: str, handle: str, httpd, httpd.themeName, httpd.maxLikeCount, httpd.maxRecentPosts, - httpd.CWlists): + httpd.CWlists, + httpd.listsEnabled): indexLines.remove(line) try: os.remove(postFilename) diff --git a/tests.py b/tests.py index 43366cb4a..f1182cb06 100644 --- a/tests.py +++ b/tests.py @@ -5773,7 +5773,7 @@ def _testWordsSimilarity() -> None: def _testAddCWfromLists(baseDir: str) -> None: print('testAddCWfromLists') translate = {} - CWlists = loadCWLists(baseDir, True, "Murdoch press") + CWlists = loadCWLists(baseDir, True) assert CWlists postJsonObject = { @@ -5783,7 +5783,7 @@ def _testAddCWfromLists(baseDir: str) -> None: "content": "" } } - addCWfromLists(postJsonObject, CWlists, translate) + addCWfromLists(postJsonObject, CWlists, translate, 'Murdoch press') assert postJsonObject['object']['sensitive'] is False assert postJsonObject['object']['summary'] is None @@ -5794,7 +5794,7 @@ def _testAddCWfromLists(baseDir: str) -> None: "content": "Blah blah news.co.uk blah blah" } } - addCWfromLists(postJsonObject, CWlists, translate) + addCWfromLists(postJsonObject, CWlists, translate, 'Murdoch press') assert postJsonObject['object']['sensitive'] is True assert postJsonObject['object']['summary'] == "Murdoch Press" @@ -5805,7 +5805,7 @@ def _testAddCWfromLists(baseDir: str) -> None: "content": "Blah blah news.co.uk blah blah" } } - addCWfromLists(postJsonObject, CWlists, translate) + addCWfromLists(postJsonObject, CWlists, translate, 'Murdoch press') assert postJsonObject['object']['sensitive'] is True assert postJsonObject['object']['summary'] == "Murdoch Press / Existing CW" diff --git a/webapp_confirm.py b/webapp_confirm.py index e51db2bd7..a2b29250d 100644 --- a/webapp_confirm.py +++ b/webapp_confirm.py @@ -36,7 +36,7 @@ def htmlConfirmDelete(cssCache: {}, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Shows a screen asking to confirm the deletion of a post """ if '/statuses/' not in messageId: @@ -81,7 +81,7 @@ def htmlConfirmDelete(cssCache: {}, peertubeInstances, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, False, False, False, False, False, False, - CWlists) + CWlists, listsEnabled) deletePostStr += '

' deletePostStr += \ '

' + \ diff --git a/webapp_frontscreen.py b/webapp_frontscreen.py index e53ac27b3..25104840a 100644 --- a/webapp_frontscreen.py +++ b/webapp_frontscreen.py @@ -35,7 +35,8 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str, CWlists: {}) -> str: + signingPrivateKeyPem: str, CWlists: {}, + listsEnabled: str) -> str: """Shows posts on the front screen of a news instance These should only be public blog posts from the features timeline which is the blog timeline of the news actor @@ -82,7 +83,7 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, maxLikeCount, False, False, False, True, False, False, - CWlists) + CWlists, listsEnabled) if postStr: profileStr += postStr + separatorStr ctr += 1 @@ -113,7 +114,7 @@ def htmlFrontScreen(signingPrivateKeyPem: str, extraJson: {}, pageNumber: int, maxItemsPerPage: int, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the news instance front screen """ nickname = profileJson['preferredUsername'] @@ -185,7 +186,7 @@ def htmlFrontScreen(signingPrivateKeyPem: str, theme, systemLanguage, maxLikeCount, signingPrivateKeyPem, - CWlists) + licenseStr + CWlists, listsEnabled) + licenseStr # Footer which is only used for system accounts profileFooterStr = ' \n' diff --git a/webapp_moderation.py b/webapp_moderation.py index 0a7298d05..21009939a 100644 --- a/webapp_moderation.py +++ b/webapp_moderation.py @@ -53,7 +53,7 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the moderation feed as html This is what you see when selecting the "mod" timeline """ @@ -73,7 +73,7 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, peertubeInstances, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, - signingPrivateKeyPem, CWlists) + signingPrivateKeyPem, CWlists, listsEnabled) def htmlAccountInfo(cssCache: {}, translate: {}, diff --git a/webapp_post.py b/webapp_post.py index ed67b537c..74d90a6b1 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1207,7 +1207,8 @@ def individualPostAsHtml(signingPrivateKeyPem: str, showPublicOnly: bool, storeToCache: bool, useCacheOnly: bool, - CWlists: {}) -> str: + CWlists: {}, + listsEnabled: str) -> str: """ Shows a single post as html """ if not postJsonObject: @@ -1654,7 +1655,7 @@ def individualPostAsHtml(signingPrivateKeyPem: str, footerStr = newFooterStr # add any content warning from the cwlists directory - addCWfromLists(postJsonObject, CWlists, translate) + addCWfromLists(postJsonObject, CWlists, translate, listsEnabled) postIsSensitive = False if postJsonObject['object'].get('sensitive'): @@ -1831,7 +1832,7 @@ def htmlIndividualPost(cssCache: {}, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show an individual post as html """ postStr = '' @@ -1885,7 +1886,7 @@ def htmlIndividualPost(cssCache: {}, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, False, authorized, False, False, False, False, - CWlists) + CWlists, listsEnabled) messageId = removeIdEnding(postJsonObject['id']) # show the previous posts @@ -1918,7 +1919,7 @@ def htmlIndividualPost(cssCache: {}, maxLikeCount, False, authorized, False, False, False, False, - CWlists) + postStr + CWlists, listsEnabled) + postStr # show the following posts postFilename = locatePost(baseDir, nickname, domain, messageId) @@ -1953,7 +1954,7 @@ def htmlIndividualPost(cssCache: {}, maxLikeCount, False, authorized, False, False, False, False, - CWlists) + CWlists, listsEnabled) cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): cssFilename = baseDir + '/epicyon.css' @@ -1977,7 +1978,8 @@ def htmlPostReplies(cssCache: {}, allowLocalNetworkAccess: bool, themeName: str, systemLanguage: str, maxLikeCount: int, - signingPrivateKeyPem: str, CWlists: {}) -> str: + signingPrivateKeyPem: str, CWlists: {}, + listsEnabled: str) -> str: """Show the replies to an individual post as html """ repliesStr = '' @@ -2001,7 +2003,7 @@ def htmlPostReplies(cssCache: {}, themeName, systemLanguage, maxLikeCount, False, False, False, False, False, False, - CWlists) + CWlists, listsEnabled) cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): diff --git a/webapp_profile.py b/webapp_profile.py index 8b2877cf8..9539f7a76 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -90,7 +90,7 @@ def htmlProfileAfterSearch(cssCache: {}, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show a profile page after a search for a fediverse address """ http = False @@ -308,7 +308,7 @@ def htmlProfileAfterSearch(cssCache: {}, allowLocalNetworkAccess, themeName, systemLanguage, maxLikeCount, False, False, False, False, False, False, - CWlists) + CWlists, listsEnabled) i += 1 if i >= 8: break @@ -520,7 +520,7 @@ def htmlProfile(signingPrivateKeyPem: str, sharedItemsFederatedDomains: [], extraJson: {}, pageNumber: int, maxItemsPerPage: int, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the profile page as html """ nickname = profileJson['preferredUsername'] @@ -543,7 +543,8 @@ def htmlProfile(signingPrivateKeyPem: str, allowLocalNetworkAccess, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, None, - pageNumber, maxItemsPerPage, CWlists) + pageNumber, maxItemsPerPage, CWlists, + listsEnabled) domain, port = getDomainFromActor(profileJson['id']) if not domain: @@ -901,7 +902,7 @@ def htmlProfile(signingPrivateKeyPem: str, theme, systemLanguage, maxLikeCount, signingPrivateKeyPem, - CWlists) + licenseStr + CWlists, listsEnabled) + licenseStr if not isGroup: if selected == 'following': profileStr += \ @@ -968,7 +969,7 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Shows posts on the profile screen These should only be public posts """ @@ -1014,7 +1015,7 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, maxLikeCount, False, False, False, True, False, False, - CWlists) + CWlists, listsEnabled) if postStr: profileStr += postStr + separatorStr ctr += 1 diff --git a/webapp_search.py b/webapp_search.py index 1f122b53d..5c48ac6b3 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -599,7 +599,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, + listsEnabled: str) -> str: """Show a page containing search results for your post history """ if historysearch.startswith("'"): @@ -685,7 +686,7 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, showIndividualPostIcons, showIndividualPostIcons, False, False, False, False, - CWlists) + CWlists, listsEnabled) if postStr: historySearchForm += separatorStr + postStr index += 1 @@ -710,7 +711,7 @@ def htmlHashtagSearch(cssCache: {}, themeName: str, systemLanguage: str, maxLikeCount: int, signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show a page containing search results for a hashtag or after selecting a hashtag from the swarm """ @@ -868,7 +869,8 @@ def htmlHashtagSearch(cssCache: {}, showRepeats, showIcons, manuallyApprovesFollowers, showPublicOnly, - storeToCache, False, CWlists) + storeToCache, False, CWlists, + listsEnabled) if postStr: hashtagSearchForm += separatorStr + postStr index += 1 diff --git a/webapp_timeline.py b/webapp_timeline.py index 9f8c35f7d..27db4313b 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -447,7 +447,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the timeline as html """ enableTimingLog = False @@ -915,7 +915,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, showIndividualPostIcons, manuallyApproveFollowers, False, True, useCacheOnly, - CWlists) + CWlists, listsEnabled) _logTimelineTiming(enableTimingLog, timelineStartTime, boxName, '12') @@ -1140,7 +1140,7 @@ def htmlShares(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the shares timeline as html """ manuallyApproveFollowers = \ @@ -1166,7 +1166,7 @@ def htmlShares(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlWanted(cssCache: {}, defaultTimeline: str, @@ -1194,7 +1194,7 @@ def htmlWanted(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the wanted timeline as html """ manuallyApproveFollowers = \ @@ -1220,7 +1220,7 @@ def htmlWanted(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInbox(cssCache: {}, defaultTimeline: str, @@ -1249,7 +1249,7 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the inbox as html """ manuallyApproveFollowers = \ @@ -1275,7 +1275,7 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlBookmarks(cssCache: {}, defaultTimeline: str, @@ -1304,7 +1304,7 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the bookmarks as html """ manuallyApproveFollowers = \ @@ -1330,7 +1330,7 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxDMs(cssCache: {}, defaultTimeline: str, @@ -1359,7 +1359,7 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the DM timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1380,7 +1380,7 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxReplies(cssCache: {}, defaultTimeline: str, @@ -1409,7 +1409,7 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the replies timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1431,7 +1431,7 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxMedia(cssCache: {}, defaultTimeline: str, @@ -1460,7 +1460,7 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the media timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1482,7 +1482,7 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, @@ -1511,7 +1511,7 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the blogs timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1533,7 +1533,7 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, @@ -1563,7 +1563,7 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the features timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1585,7 +1585,7 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlInboxNews(cssCache: {}, defaultTimeline: str, @@ -1614,7 +1614,7 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the news timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1636,7 +1636,7 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) def htmlOutbox(cssCache: {}, defaultTimeline: str, @@ -1665,7 +1665,7 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, maxLikeCount: int, sharedItemsFederatedDomains: [], signingPrivateKeyPem: str, - CWlists: {}) -> str: + CWlists: {}, listsEnabled: str) -> str: """Show the Outbox as html """ manuallyApproveFollowers = \ @@ -1688,4 +1688,4 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, allowLocalNetworkAccess, textModeBanner, accessKeys, systemLanguage, maxLikeCount, sharedItemsFederatedDomains, signingPrivateKeyPem, - CWlists) + CWlists, listsEnabled) From 48d3235b8fff8ffabfc88abf2fef3c254da257ae Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:03:26 +0100 Subject: [PATCH 21/27] Check that lists are enabled --- webapp_profile.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 9539f7a76..6b2af9093 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1677,17 +1677,18 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 200, '', False) CWlistsStr = '' - print('CWlistsStr 0: ' + str(CWlists)) - for name, item in CWlists.items(): - print('CWlistsStr: ' + name) - variableName = 'list' + name.replace(' ', '').replace("'", '') - if name in listsEnabled: - listIsEnabled = True - else: - listIsEnabled = False - if translate.get(name): - name = translate[name] - CWlistsStr += editCheckBox(name, variableName, listIsEnabled) + if listsEnabled: + print('CWlistsStr 0: ' + str(CWlists)) + for name, item in CWlists.items(): + print('CWlistsStr: ' + name) + variableName = 'list' + name.replace(' ', '').replace("'", '') + if name in listsEnabled: + listIsEnabled = True + else: + listIsEnabled = False + if translate.get(name): + name = translate[name] + CWlistsStr += editCheckBox(name, variableName, listIsEnabled) if CWlistsStr: idx = 'Add content warnings for the following sites' editProfileForm += \ From 9e6f608c91721710700bd891bad151892175b8e5 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:05:38 +0100 Subject: [PATCH 22/27] Checking for enabled lists --- webapp_profile.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 6b2af9093..f36b28e73 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1677,18 +1677,15 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 200, '', False) CWlistsStr = '' - if listsEnabled: - print('CWlistsStr 0: ' + str(CWlists)) - for name, item in CWlists.items(): - print('CWlistsStr: ' + name) - variableName = 'list' + name.replace(' ', '').replace("'", '') + for name, item in CWlists.items(): + variableName = 'list' + name.replace(' ', '').replace("'", '') + listIsEnabled = False + if listsEnabled: if name in listsEnabled: listIsEnabled = True - else: - listIsEnabled = False - if translate.get(name): - name = translate[name] - CWlistsStr += editCheckBox(name, variableName, listIsEnabled) + if translate.get(name): + name = translate[name] + CWlistsStr += editCheckBox(name, variableName, listIsEnabled) if CWlistsStr: idx = 'Add content warnings for the following sites' editProfileForm += \ From f8a7622b6b4d4e5e90ef436b7de474df6cbc3254 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:07:08 +0100 Subject: [PATCH 23/27] Indentation --- webapp_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index f36b28e73..43ab56f3f 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1685,7 +1685,7 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, listIsEnabled = True if translate.get(name): name = translate[name] - CWlistsStr += editCheckBox(name, variableName, listIsEnabled) + CWlistsStr += editCheckBox(name, variableName, listIsEnabled) if CWlistsStr: idx = 'Add content warnings for the following sites' editProfileForm += \ From d8b2c396e97735446feec9c3afe2cd9bcacb4b84 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:08:28 +0100 Subject: [PATCH 24/27] Extra line --- webapp_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index 43ab56f3f..0f69b39fb 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1690,7 +1690,7 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, idx = 'Add content warnings for the following sites' editProfileForm += \ '\n' + \ - CWlistsStr + '
' + CWlistsStr editProfileForm += endEditSection() return editProfileForm From d2a4444c7e4c4413b0cd3b06b433099ee9bb904c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:19:44 +0100 Subject: [PATCH 25/27] Tidying --- blocking.py | 6 ++++++ webapp_profile.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/blocking.py b/blocking.py index 0ba2d3623..67952d27b 100644 --- a/blocking.py +++ b/blocking.py @@ -959,3 +959,9 @@ def addCWfromLists(postJsonObject: {}, CWlists: {}, translate: {}, if cw: postJsonObject['object']['summary'] = cw postJsonObject['object']['sensitive'] = True + + +def getCWlistVariable(listName: str) -> str: + """Returns the variable associated with a CW list + """ + return 'list' + listName.replace(' ', '').replace("'", '') diff --git a/webapp_profile.py b/webapp_profile.py index 0f69b39fb..5617db465 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -69,6 +69,7 @@ from webapp_utils import endEditSection from blog import getBlogAddress from webapp_post import individualPostAsHtml from webapp_timeline import htmlIndividualShare +from blocking import getCWlistVariable def htmlProfileAfterSearch(cssCache: {}, @@ -1678,7 +1679,7 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, CWlistsStr = '' for name, item in CWlists.items(): - variableName = 'list' + name.replace(' ', '').replace("'", '') + variableName = getCWlistVariable(name) listIsEnabled = False if listsEnabled: if name in listsEnabled: From 2ad58b798ef19dfc460cf206c94b0fefbb711d87 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:29:08 +0100 Subject: [PATCH 26/27] Update selected cw lists --- daemon.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/daemon.py b/daemon.py index c24451bcb..ca17a2845 100644 --- a/daemon.py +++ b/daemon.py @@ -125,6 +125,7 @@ from media import replaceTwitter from media import attachMedia from media import pathIsVideo from media import pathIsAudio +from blocking import getCWlistVariable from blocking import loadCWLists from blocking import updateBlockedCache from blocking import mutePost @@ -5698,6 +5699,22 @@ class PubServer(BaseHTTPRequestHandler): # save blocked user agents # This is admin lebel and global to the instance if path.startswith('/users/' + adminNickname + '/'): + # set selected content warning lists + newListsEnabled = '' + for name, item in self.server.CWlists.items(): + listVarName = getCWlistVariable(name) + if fields.get(listVarName): + if fields[listVarName] == 'on': + if newListsEnabled: + newListsEnabled += ', ' + name + else: + newListsEnabled += name + if newListsEnabled != self.server.listsEnabled: + self.server.listsEnabled = newListsEnabled + setConfigParam(self.server.baseDir, + "listsEnabled", + newListsEnabled) + userAgentsBlocked = [] if fields.get('userAgentsBlockedStr'): userAgentsBlockedStr = \ From 88c4b14ca3b83caff4095415c2aa8d363f045d28 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 21 Oct 2021 20:45:39 +0100 Subject: [PATCH 27/27] Updating enabled cw lists --- daemon.py | 5 ++--- webapp_profile.py | 50 ++++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/daemon.py b/daemon.py index ca17a2845..9bbbf2c09 100644 --- a/daemon.py +++ b/daemon.py @@ -5696,9 +5696,7 @@ class PubServer(BaseHTTPRequestHandler): except BaseException: pass - # save blocked user agents - # This is admin lebel and global to the instance - if path.startswith('/users/' + adminNickname + '/'): + if isModerator(self.server.baseDir, nickname): # set selected content warning lists newListsEnabled = '' for name, item in self.server.CWlists.items(): @@ -5715,6 +5713,7 @@ class PubServer(BaseHTTPRequestHandler): "listsEnabled", newListsEnabled) + # save blocked user agents userAgentsBlocked = [] if fields.get('userAgentsBlockedStr'): userAgentsBlockedStr = \ diff --git a/webapp_profile.py b/webapp_profile.py index 5617db465..914d212c8 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -34,6 +34,7 @@ from person import personBoxJson from person import getActorJson from person import getPersonAvatarUrl from webfinger import webfingerHandle +from posts import isModerator from posts import parseUserFeed from posts import getPersonBox from posts import isCreateInsideAnnounce @@ -1667,31 +1668,32 @@ def _htmlEditProfileFiltering(baseDir: str, nickname: str, domain: str, 'style="height:200px" spellcheck="false">' + \ allowedInstancesStr + '\n' - userAgentsBlockedStr = '' - for ua in userAgentsBlocked: - if userAgentsBlockedStr: - userAgentsBlockedStr += '\n' - userAgentsBlockedStr += ua - editProfileForm += \ - editTextArea(translate['Blocked User Agents'], - 'userAgentsBlockedStr', userAgentsBlockedStr, - 200, '', False) - - CWlistsStr = '' - for name, item in CWlists.items(): - variableName = getCWlistVariable(name) - listIsEnabled = False - if listsEnabled: - if name in listsEnabled: - listIsEnabled = True - if translate.get(name): - name = translate[name] - CWlistsStr += editCheckBox(name, variableName, listIsEnabled) - if CWlistsStr: - idx = 'Add content warnings for the following sites' + if isModerator(baseDir, nickname): + userAgentsBlockedStr = '' + for ua in userAgentsBlocked: + if userAgentsBlockedStr: + userAgentsBlockedStr += '\n' + userAgentsBlockedStr += ua editProfileForm += \ - '\n' + \ - '
' + CWlistsStr + editTextArea(translate['Blocked User Agents'], + 'userAgentsBlockedStr', userAgentsBlockedStr, + 200, '', False) + + CWlistsStr = '' + for name, item in CWlists.items(): + variableName = getCWlistVariable(name) + listIsEnabled = False + if listsEnabled: + if name in listsEnabled: + listIsEnabled = True + if translate.get(name): + name = translate[name] + CWlistsStr += editCheckBox(name, variableName, listIsEnabled) + if CWlistsStr: + idx = 'Add content warnings for the following sites' + editProfileForm += \ + '\n' + \ + '
' + CWlistsStr editProfileForm += endEditSection() return editProfileForm