From 429044c937dae1c55eb6ab86dfdd61a729c0e31f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:01:58 +0000 Subject: [PATCH 01/39] Ensure the right number of posts per page --- posts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/posts.py b/posts.py index ff00c98a1..ab4fca28b 100644 --- a/posts.py +++ b/posts.py @@ -3148,7 +3148,8 @@ def _createBoxIndexed(recentPostsCache: {}, if os.path.isfile(indexFilename): maxPostCtr = itemsPerPage * pageNumber with open(indexFilename, 'r') as indexFile: - while postsCtr < maxPostCtr: + postsAddedToTimeline = 0 + while postsAddedToTimeline < itemsPerPage: postFilename = indexFile.readline() if not postFilename: @@ -3217,6 +3218,7 @@ def _createBoxIndexed(recentPostsCache: {}, boxname, postsInBox, boxActor) postsCtr += 1 + postsAddedToTimeline +=1 continue # read the post from file @@ -3226,6 +3228,7 @@ def _createBoxIndexed(recentPostsCache: {}, if fullPostFilename: _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor) + postsAddedToTimeline +=1 else: if timelineNickname != nickname: # if this is the features timeline @@ -3235,6 +3238,7 @@ def _createBoxIndexed(recentPostsCache: {}, if fullPostFilename: _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor) + postsAddedToTimeline +=1 else: print('WARN: features timeline. ' + 'Unable to locate post ' + postUrl) From b18d5b6242e8ed43608eca940a9613dad79536e0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:18:32 +0000 Subject: [PATCH 02/39] Ensure that items are counted as being added to the timeline --- posts.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/posts.py b/posts.py index ab4fca28b..2da756b40 100644 --- a/posts.py +++ b/posts.py @@ -3146,7 +3146,6 @@ def _createBoxIndexed(recentPostsCache: {}, '/' + indexBoxName + '.index' postsCtr = 0 if os.path.isfile(indexFilename): - maxPostCtr = itemsPerPage * pageNumber with open(indexFilename, 'r') as indexFile: postsAddedToTimeline = 0 while postsAddedToTimeline < itemsPerPage: @@ -3214,21 +3213,22 @@ def _createBoxIndexed(recentPostsCache: {}, if postUrl in recentPostsCache['index']: if recentPostsCache['json'].get(postUrl): url = recentPostsCache['json'][postUrl] - _addPostStringToTimeline(url, - boxname, postsInBox, - boxActor) - postsCtr += 1 - postsAddedToTimeline +=1 - continue + if _addPostStringToTimeline(url, + boxname, postsInBox, + boxActor): + postsCtr += 1 + postsAddedToTimeline += 1 + continue # read the post from file fullPostFilename = \ locatePost(baseDir, nickname, domain, postUrl, False) if fullPostFilename: - _addPostToTimeline(fullPostFilename, boxname, - postsInBox, boxActor) - postsAddedToTimeline +=1 + if _addPostToTimeline(fullPostFilename, boxname, + postsInBox, boxActor): + postsAddedToTimeline += 1 + postsCtr += 1 else: if timelineNickname != nickname: # if this is the features timeline @@ -3236,9 +3236,10 @@ def _createBoxIndexed(recentPostsCache: {}, locatePost(baseDir, timelineNickname, domain, postUrl, False) if fullPostFilename: - _addPostToTimeline(fullPostFilename, boxname, - postsInBox, boxActor) - postsAddedToTimeline +=1 + if _addPostToTimeline(fullPostFilename, boxname, + postsInBox, boxActor): + postsAddedToTimeline += 1 + postsCtr += 1 else: print('WARN: features timeline. ' + 'Unable to locate post ' + postUrl) @@ -3246,8 +3247,6 @@ def _createBoxIndexed(recentPostsCache: {}, print('WARN: Unable to locate post ' + postUrl + ' nickname ' + nickname) - postsCtr += 1 - # Generate first and last entries within header if postsCtr > 0: lastPage = int(postsCtr / itemsPerPage) From 8689b634d6ee7ee131e512e7a2fd689121c677e5 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:39:03 +0000 Subject: [PATCH 03/39] Ignore updates or likes --- posts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/posts.py b/posts.py index 2da756b40..79bd1149b 100644 --- a/posts.py +++ b/posts.py @@ -3027,6 +3027,9 @@ def _addPostStringToTimeline(postStr: str, boxname: str, postsInBox: [], boxActor: str) -> bool: """ is this a valid timeline post? """ + if '"Update"' in postStr or '"Like"' in postStr: + return False + # must be a recognized ActivityPub type if ('"Note"' in postStr or '"EncryptedMessage"' in postStr or From 3d3dc3c7f5663211e1ddea03883e310e4abcb739 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:45:03 +0000 Subject: [PATCH 04/39] Only show creates or announces --- webapp_timeline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webapp_timeline.py b/webapp_timeline.py index be498f826..4ad4c92b0 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -672,8 +672,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, # show each post in the timeline for item in timelineJson['orderedItems']: if item['type'] == 'Create' or \ - item['type'] == 'Announce' or \ - item['type'] == 'Update': + item['type'] == 'Announce': # is the actor who sent this post snoozed? if isPersonSnoozed(baseDir, nickname, domain, item['actor']): continue From 2ac894598269f3a540eb0a7bd3e09de1ab4dda48 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:48:37 +0000 Subject: [PATCH 05/39] Debug --- webapp_timeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webapp_timeline.py b/webapp_timeline.py index 4ad4c92b0..32a7ca6e1 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -695,6 +695,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, _logTimelineTiming(enableTimingLog, timelineStartTime, boxName, '10') + else: + print('Muted post in timeline ' + boxName) if not currTlStr: _logTimelineTiming(enableTimingLog, From 2de18f89f2f22fd0f5e58072b112d75ba127cdd2 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 15:59:39 +0000 Subject: [PATCH 06/39] Debug --- posts.py | 3 +++ webapp_timeline.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/posts.py b/posts.py index 79bd1149b..ccecc4027 100644 --- a/posts.py +++ b/posts.py @@ -3250,6 +3250,9 @@ def _createBoxIndexed(recentPostsCache: {}, print('WARN: Unable to locate post ' + postUrl + ' nickname ' + nickname) + print('Posts added to timeline ' + boxname + ': ' + + str(postsAddedToTimeline)) + # Generate first and last entries within header if postsCtr > 0: lastPage = int(postsCtr / itemsPerPage) diff --git a/webapp_timeline.py b/webapp_timeline.py index 32a7ca6e1..a8be5ca20 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -731,10 +731,12 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, itemCtr += 1 tlStr += textModeSeparator + currTlStr if separatorStr: - tlStr += separatorStr + tlStr += separatorStr if boxName == 'tlmedia': tlStr += '\n' + print('Items in timeline ' + boxName + ': ' + str(itemCtr)) + # page down arrow if itemCtr > 2: tlStr += textModeSeparator From 0e208f4121aeb9d11ed9fc6e2612102c252819f8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 16:03:56 +0000 Subject: [PATCH 07/39] More debug --- webapp_timeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp_timeline.py b/webapp_timeline.py index a8be5ca20..0b946fcc2 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -735,7 +735,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, if boxName == 'tlmedia': tlStr += '\n' - print('Items in timeline ' + boxName + ': ' + str(itemCtr)) + print('Items in timeline ' + boxName + ': ' + + str(itemCtr) + ' ' + str(timelineJson['orderedItems'])) # page down arrow if itemCtr > 2: From 9caa658a5ec6fcfad41e6102315e759d5ad34187 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 16:10:25 +0000 Subject: [PATCH 08/39] Only add debug if there are a small number of posts --- posts.py | 5 +++-- webapp_timeline.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/posts.py b/posts.py index ccecc4027..c0cc447a9 100644 --- a/posts.py +++ b/posts.py @@ -3250,8 +3250,9 @@ def _createBoxIndexed(recentPostsCache: {}, print('WARN: Unable to locate post ' + postUrl + ' nickname ' + nickname) - print('Posts added to timeline ' + boxname + ': ' + - str(postsAddedToTimeline)) + if postsCtr < 3: + print('Posts added to json timeline ' + boxname + ': ' + + str(postsAddedToTimeline)) # Generate first and last entries within header if postsCtr > 0: diff --git a/webapp_timeline.py b/webapp_timeline.py index 0b946fcc2..6e2ec136b 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -735,9 +735,10 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, if boxName == 'tlmedia': tlStr += '\n' - print('Items in timeline ' + boxName + ': ' + - str(itemCtr) + ' ' + str(timelineJson['orderedItems'])) - + if itemCtr < 3: + print('Items added to html timeline ' + boxName + ': ' + + str(itemCtr) + ' ' + str(timelineJson['orderedItems'])) + # page down arrow if itemCtr > 2: tlStr += textModeSeparator From 79610c8bf89e95e925064ba91cb0381e48a7dd67 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 16:48:28 +0000 Subject: [PATCH 09/39] Be more generous with post recency --- newswire.py | 2 +- utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newswire.py b/newswire.py index 29e24e3b7..d54bcfe22 100644 --- a/newswire.py +++ b/newswire.py @@ -148,7 +148,7 @@ def _validFeedDate(pubDate: str) -> bool: # convert from YY-MM-DD HH:MM:SS+00:00 to # YY-MM-DDTHH:MM:SSZ postDate = pubDate.replace(' ', 'T').replace('+00:00', 'Z') - return validPostDate(postDate, 30) + return validPostDate(postDate, 90) def parseFeedDate(pubDate: str) -> str: diff --git a/utils.py b/utils.py index cad02295d..e12f23064 100644 --- a/utils.py +++ b/utils.py @@ -100,7 +100,7 @@ def hasUsersPath(pathStr: str) -> bool: return False -def validPostDate(published: str, maxAgeDays=7) -> bool: +def validPostDate(published: str, maxAgeDays=90) -> bool: """Returns true if the published date is recent and is not in the future """ baselineTime = datetime.datetime(1970, 1, 1) From 568d73845dc685d18c39995bd614ac93031a324e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 18:03:15 +0000 Subject: [PATCH 10/39] Mark announces which are invalid with reject extension --- posts.py | 61 +++++++++++++++++++++++++++++++++++----------- utils.py | 2 +- webapp_timeline.py | 2 +- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/posts.py b/posts.py index c0cc447a9..20dac08e4 100644 --- a/posts.py +++ b/posts.py @@ -3157,6 +3157,10 @@ def _createBoxIndexed(recentPostsCache: {}, if not postFilename: break + # has the post been rejected? + if os.path.isfile(postFilename + '.reject'): + continue + # apply votes within this timeline if newswireVotesThreshold > 0: # note that the presence of an arrival file also indicates @@ -3875,9 +3879,21 @@ def populateRepliesJson(baseDir: str, nickname: str, domain: str, repliesJson['orderedItems'].append(pjo) -def _rejectAnnounce(announceFilename: str): +def _rejectAnnounce(announceFilename: str, + baseDir: str, nickname: str, domain: str, + announcePostId: str): """Marks an announce as rejected """ + # reject the announce activity + announcePostFilename = \ + locatePost(baseDir, nickname, domain, announcePostId) + if announcePostFilename: + rejectAnnounceFile = open(announcePostFilename + '.reject', "w+") + if rejectAnnounceFile: + rejectAnnounceFile.write('\n') + rejectAnnounceFile.close() + + # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): rejectAnnounceFile = open(announceFilename + '.reject', "w+") if rejectAnnounceFile: @@ -3901,6 +3917,10 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, announceCacheDir = baseDir + '/cache/announce/' + nickname if not os.path.isdir(announceCacheDir): os.mkdir(announceCacheDir) + + postId = None + if postJsonObject.get('id'): + postId = postJsonObject['id'] announceFilename = \ announceCacheDir + '/' + \ postJsonObject['object'].replace('/', '#') + '.json' @@ -3961,43 +3981,54 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, if not isinstance(announcedJson, dict): print('WARN: announce json is not a dict - ' + postJsonObject['object']) - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not announcedJson.get('id'): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if '/statuses/' not in announcedJson['id']: - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not hasUsersPath(announcedJson['id']): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not announcedJson.get('type'): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if announcedJson['type'] != 'Note' and \ announcedJson['type'] != 'Article': # You can only announce Note or Article types - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not announcedJson.get('content'): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not announcedJson.get('published'): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if not validPostDate(announcedJson['published']): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None # Check the content of the announce contentStr = announcedJson['content'] if dangerousMarkup(contentStr, allowLocalNetworkAccess): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None if isFiltered(baseDir, nickname, domain, contentStr): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None # remove any long words @@ -4016,7 +4047,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, announcedJson) if announcedJson['type'] != 'Create': # Create wrap failed - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None # labelAccusatoryPost(postJsonObject, translate) @@ -4032,7 +4064,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, attributedDomain = getFullDomain(attributedDomain, attributedPort) if isBlocked(baseDir, nickname, domain, attributedNickname, attributedDomain): - _rejectAnnounce(announceFilename) + _rejectAnnounce(announceFilename, + baseDir, nickname, domain, postId) return None postJsonObject = announcedJson replaceYouTube(postJsonObject, YTReplacementDomain) diff --git a/utils.py b/utils.py index e12f23064..859b138d8 100644 --- a/utils.py +++ b/utils.py @@ -1206,7 +1206,7 @@ def deletePost(baseDir: str, httpPrefix: str, # remove any attachment _removeAttachment(baseDir, httpPrefix, domain, postJsonObject) - extensions = ('votes', 'arrived', 'muted', '.tts') + extensions = ('votes', 'arrived', 'muted', 'tts', 'reject') for ext in extensions: extFilename = postFilename + '.' + ext if os.path.isfile(extFilename): diff --git a/webapp_timeline.py b/webapp_timeline.py index 6e2ec136b..71e88d932 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -731,7 +731,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, itemCtr += 1 tlStr += textModeSeparator + currTlStr if separatorStr: - tlStr += separatorStr + tlStr += separatorStr if boxName == 'tlmedia': tlStr += '\n' From 7b92186deb5e39f88d34a1a0107ec5248c931a59 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 18:52:36 +0000 Subject: [PATCH 11/39] Mark failed announce download as rejected --- webapp_post.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/webapp_post.py b/webapp_post.py index c06c55959..2dfee6d26 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1290,6 +1290,15 @@ def individualPostAsHtml(allowDownloads: bool, YTReplacementDomain, allowLocalNetworkAccess) if not postJsonAnnounce: + # if the announce could not be downloaded then mark it as rejected + announceFilename = \ + locatePost(baseDir, nickname, domain, + postJsonObject['id']) + if announceFilename: + rejectFile = open(announceFilename + '.reject', "w+") + if rejectFile: + rejectFile.write('\n') + rejectFile.close() return '' postJsonObject = postJsonAnnounce From e1de415809dae68fc9a394388171390aa4a5ba86 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 19:00:37 +0000 Subject: [PATCH 12/39] Tidying --- posts.py | 10 ++-------- utils.py | 13 +++++++++++++ webapp_post.py | 10 ++-------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/posts.py b/posts.py index 20dac08e4..05e68a497 100644 --- a/posts.py +++ b/posts.py @@ -31,6 +31,7 @@ from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader from siteactive import siteIsActive +from utils import rejectPostId from utils import removeInvalidChars from utils import fileLastModified from utils import isPublicPost @@ -3884,14 +3885,7 @@ def _rejectAnnounce(announceFilename: str, announcePostId: str): """Marks an announce as rejected """ - # reject the announce activity - announcePostFilename = \ - locatePost(baseDir, nickname, domain, announcePostId) - if announcePostFilename: - rejectAnnounceFile = open(announcePostFilename + '.reject', "w+") - if rejectAnnounceFile: - rejectAnnounceFile.write('\n') - rejectAnnounceFile.close() + rejectPostId(baseDir, nickname, domain, announcePostId) # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): diff --git a/utils.py b/utils.py index 859b138d8..33535dc78 100644 --- a/utils.py +++ b/utils.py @@ -2034,3 +2034,16 @@ def camelCaseSplit(text: str) -> str: for word in matches: resultStr += word.group(0) + ' ' return resultStr.strip() + + +def rejectPostId(baseDir: str, nickname: str, domain: str, + postId: str) -> None: + """ Marks the given post as rejected + """ + postFilename = locatePost(baseDir, nickname, domain, postId) + if not postFilename: + return + rejectFile = open(postFilename + '.reject', "w+") + if rejectFile: + rejectFile.write('\n') + rejectFile.close() diff --git a/webapp_post.py b/webapp_post.py index 2dfee6d26..035f8bddb 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -22,6 +22,7 @@ from posts import getPersonBox from posts import isDM from posts import downloadAnnounce from posts import populateRepliesJson +from utils import rejectPostId from utils import isRecentPost from utils import getConfigParam from utils import getFullDomain @@ -1291,14 +1292,7 @@ def individualPostAsHtml(allowDownloads: bool, allowLocalNetworkAccess) if not postJsonAnnounce: # if the announce could not be downloaded then mark it as rejected - announceFilename = \ - locatePost(baseDir, nickname, domain, - postJsonObject['id']) - if announceFilename: - rejectFile = open(announceFilename + '.reject', "w+") - if rejectFile: - rejectFile.write('\n') - rejectFile.close() + rejectPostId(baseDir, nickname, domain, postJsonObject['id']) return '' postJsonObject = postJsonAnnounce From 151fa8cab6d67699b54b2f3942ae00599eedd59c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 19:23:33 +0000 Subject: [PATCH 13/39] Remove rejected posts from recent posts cache --- inbox.py | 6 ++++-- posts.py | 49 ++++++++++++++++++++++++++++++++----------------- utils.py | 25 +++++++++++++++++++++++-- webapp_post.py | 6 ++++-- 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/inbox.py b/inbox.py index 3e9bd1caa..89bae2036 100644 --- a/inbox.py +++ b/inbox.py @@ -1372,7 +1372,8 @@ def _receiveAnnounce(recentPostsCache: {}, messageJson, __version__, translate, YTReplacementDomain, - allowLocalNetworkAccess) + allowLocalNetworkAccess, + recentPostsCache) if not postJsonObject: notInOnion = True if onionDomain: @@ -2450,7 +2451,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, if isImageMedia(session, baseDir, httpPrefix, nickname, domain, postJsonObject, translate, YTReplacementDomain, - allowLocalNetworkAccess): + allowLocalNetworkAccess, + recentPostsCache): # media index will be updated updateIndexList.append('tlmedia') if isBlogPost(postJsonObject): diff --git a/posts.py b/posts.py index 05e68a497..c85d4370e 100644 --- a/posts.py +++ b/posts.py @@ -2953,7 +2953,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, postJsonObject: {}, translate: {}, YTReplacementDomain: str, - allowLocalNetworkAccess: bool) -> bool: + allowLocalNetworkAccess: bool, + recentPostsCache: {}) -> bool: """Returns true if the given post has attached image media """ if postJsonObject['type'] == 'Announce': @@ -2962,7 +2963,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str, nickname, domain, postJsonObject, __version__, translate, YTReplacementDomain, - allowLocalNetworkAccess) + allowLocalNetworkAccess, + recentPostsCache) if postJsonAnnounce: postJsonObject = postJsonAnnounce if postJsonObject['type'] != 'Create': @@ -3882,10 +3884,10 @@ def populateRepliesJson(baseDir: str, nickname: str, domain: str, def _rejectAnnounce(announceFilename: str, baseDir: str, nickname: str, domain: str, - announcePostId: str): + announcePostId: str, recentPostsCache: {}): """Marks an announce as rejected """ - rejectPostId(baseDir, nickname, domain, announcePostId) + rejectPostId(baseDir, nickname, domain, announcePostId, recentPostsCache) # reject the post referenced by the announce activity object if not os.path.isfile(announceFilename + '.reject'): @@ -3899,7 +3901,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, postJsonObject: {}, projectVersion: str, translate: {}, YTReplacementDomain: str, - allowLocalNetworkAccess: bool) -> {}: + allowLocalNetworkAccess: bool, + recentPostsCache: {}) -> {}: """Download the post referenced by an announce """ if not postJsonObject.get('object'): @@ -3976,11 +3979,13 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, print('WARN: announce json is not a dict - ' + postJsonObject['object']) _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not announcedJson.get('id'): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if '/statuses/' not in announcedJson['id']: _rejectAnnounce(announceFilename, @@ -3988,41 +3993,49 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, return None if not hasUsersPath(announcedJson['id']): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not announcedJson.get('type'): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if announcedJson['type'] != 'Note' and \ announcedJson['type'] != 'Article': # You can only announce Note or Article types _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not announcedJson.get('content'): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not announcedJson.get('published'): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not validPostDate(announcedJson['published']): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None # Check the content of the announce contentStr = announcedJson['content'] if dangerousMarkup(contentStr, allowLocalNetworkAccess): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if isFiltered(baseDir, nickname, domain, contentStr): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None # remove any long words @@ -4042,7 +4055,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, if announcedJson['type'] != 'Create': # Create wrap failed _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None # labelAccusatoryPost(postJsonObject, translate) @@ -4059,7 +4073,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, if isBlocked(baseDir, nickname, domain, attributedNickname, attributedDomain): _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None postJsonObject = announcedJson replaceYouTube(postJsonObject, YTReplacementDomain) diff --git a/utils.py b/utils.py index 33535dc78..57d41662f 100644 --- a/utils.py +++ b/utils.py @@ -2037,12 +2037,33 @@ def camelCaseSplit(text: str) -> str: def rejectPostId(baseDir: str, nickname: str, domain: str, - postId: str) -> None: - """ Marks the given post as rejected + postId: str, recentPostsCache: {}) -> None: + """ Marks the given post as rejected, + for example an announce which is too old """ postFilename = locatePost(baseDir, nickname, domain, postId) if not postFilename: return + + if recentPostsCache.get('index'): + # if this is a full path then remove the directories + indexFilename = postFilename + if '/' in postFilename: + indexFilename = postFilename.split('/')[-1] + + # filename of the post without any extension or path + # This should also correspond to any index entry in + # the posts cache + postUrl = \ + indexFilename.replace('\n', '').replace('\r', '') + postUrl = postUrl.replace('.json', '').strip() + + if postUrl in recentPostsCache['index']: + if recentPostsCache['json'].get(postUrl): + del recentPostsCache['json'][postUrl] + if recentPostsCache['html'].get(postUrl): + del recentPostsCache['html'][postUrl] + rejectFile = open(postFilename + '.reject', "w+") if rejectFile: rejectFile.write('\n') diff --git a/webapp_post.py b/webapp_post.py index 035f8bddb..532f535d2 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1289,10 +1289,12 @@ def individualPostAsHtml(allowDownloads: bool, nickname, domain, postJsonObject, projectVersion, translate, YTReplacementDomain, - allowLocalNetworkAccess) + allowLocalNetworkAccess, + recentPostsCache) if not postJsonAnnounce: # if the announce could not be downloaded then mark it as rejected - rejectPostId(baseDir, nickname, domain, postJsonObject['id']) + rejectPostId(baseDir, nickname, domain, postJsonObject['id'], + recentPostsCache) return '' postJsonObject = postJsonAnnounce From f78fae40313f98425e879d3cfe68feb6acb7db6b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 19:29:09 +0000 Subject: [PATCH 14/39] Use full filename for post reject --- posts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/posts.py b/posts.py index c85d4370e..de2c06953 100644 --- a/posts.py +++ b/posts.py @@ -3160,10 +3160,6 @@ def _createBoxIndexed(recentPostsCache: {}, if not postFilename: break - # has the post been rejected? - if os.path.isfile(postFilename + '.reject'): - continue - # apply votes within this timeline if newswireVotesThreshold > 0: # note that the presence of an arrival file also indicates @@ -3235,6 +3231,10 @@ def _createBoxIndexed(recentPostsCache: {}, locatePost(baseDir, nickname, domain, postUrl, False) if fullPostFilename: + # has the post been rejected? + if os.path.isfile(fullPostFilename + '.reject'): + continue + if _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor): postsAddedToTimeline += 1 From 5ef83756c37581f8a1e480cb5e87d123ec2b02cc Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 19:37:58 +0000 Subject: [PATCH 15/39] Missing parameter --- posts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/posts.py b/posts.py index de2c06953..d944494bd 100644 --- a/posts.py +++ b/posts.py @@ -3234,7 +3234,7 @@ def _createBoxIndexed(recentPostsCache: {}, # has the post been rejected? if os.path.isfile(fullPostFilename + '.reject'): continue - + if _addPostToTimeline(fullPostFilename, boxname, postsInBox, boxActor): postsAddedToTimeline += 1 @@ -3989,7 +3989,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, return None if '/statuses/' not in announcedJson['id']: _rejectAnnounce(announceFilename, - baseDir, nickname, domain, postId) + baseDir, nickname, domain, postId, + recentPostsCache) return None if not hasUsersPath(announcedJson['id']): _rejectAnnounce(announceFilename, From f7e237858154cc892f0fad31e49b4b1a57dc74dd Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 5 Mar 2021 20:13:35 +0000 Subject: [PATCH 16/39] Tidying --- posts.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/posts.py b/posts.py index d944494bd..dbf0dc2fe 100644 --- a/posts.py +++ b/posts.py @@ -3030,9 +3030,6 @@ def _addPostStringToTimeline(postStr: str, boxname: str, postsInBox: [], boxActor: str) -> bool: """ is this a valid timeline post? """ - if '"Update"' in postStr or '"Like"' in postStr: - return False - # must be a recognized ActivityPub type if ('"Note"' in postStr or '"EncryptedMessage"' in postStr or From 17624f6eece8fc4b437f71799619dd9d72d5c780 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 6 Mar 2021 11:25:28 +0000 Subject: [PATCH 17/39] More clarity about what type of accounts --- translations/en.json | 2 +- translations/oc.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/translations/en.json b/translations/en.json index e4c98b58a..33fa210f3 100644 --- a/translations/en.json +++ b/translations/en.json @@ -360,7 +360,7 @@ "New account": "New account", "Moved to new account address": "Moved to new account address", "Yet another Epicyon Instance": "Yet another Epicyon Instance", - "Other accounts": "Other accounts", + "Other accounts": "Other fediverse accounts", "Pin this post to your profile.": "Pin this post to your profile.", "Administered by": "Administered by", "Version": "Version", diff --git a/translations/oc.json b/translations/oc.json index bf8bac91f..420e23b0b 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -356,7 +356,7 @@ "New account": "New account", "Moved to new account address": "Moved to new account address", "Yet another Epicyon Instance": "Yet another Epicyon Instance", - "Other accounts": "Other accounts", + "Other accounts": "Other fediverse accounts", "Pin this post to your profile.": "Pin this post to your profile.", "Administered by": "Administered by", "Version": "Version", From c8a0e2d9201c9313c11c9aeeca8d338c3f23619c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 6 Mar 2021 11:42:29 +0000 Subject: [PATCH 18/39] Re-arrange edit profile screen sections --- webapp_profile.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 4a1dff572..824796e58 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1211,6 +1211,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, '

' + skillsStr += '\n' cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): @@ -1460,6 +1461,18 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, editProfileForm += \ ' \n' + editProfileForm += '
\n' + editProfileForm += \ + ' \n' + editProfileForm += '\n' + + # Contact information + editProfileForm = '
' + editProfileForm += '
\n' + editProfileForm += \ + ' \n' editProfileForm += \ '
\n' editProfileForm += \ @@ -1476,11 +1489,6 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' \n' - editProfileForm += '
\n' - editProfileForm += \ - ' \n' - editProfileForm += '
\n' editProfileForm += \ ' \n' - editProfileForm += '
\n' - editProfileForm += \ - ' \n' editProfileForm += \ '
\n' @@ -1618,7 +1622,10 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' ' + \ translate["Don't show the Like button"] + '
\n' + editProfileForm += '
\n' + # Content controls + editProfileForm += '
\n' editProfileForm += \ '
\n' @@ -1711,9 +1718,12 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, 'teams with an appropriate combination of skills.' editProfileForm += ' \n' - editProfileForm += skillsStr + themesDropdown - editProfileForm += moderatorsStr + editorsStr + peertubeStr - editProfileForm += '
\n' + instanceStr + editProfileForm += skillsStr + editProfileForm += '
\n' + editProfileForm += themesDropdown + moderatorsStr + editProfileForm += editorsStr + peertubeStr + editProfileForm += '
\n' + editProfileForm += instanceStr editProfileForm += '
\n' editProfileForm += '
\n' From 11f44e62b0197ba31dfeb70d0ca9ef8ff461f392 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 6 Mar 2021 11:44:28 +0000 Subject: [PATCH 19/39] Typo --- webapp_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index 824796e58..5f0d868da 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1468,7 +1468,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, editProfileForm += '
\n' # Contact information - editProfileForm = '
' + editProfileForm += '
' editProfileForm += '
\n' editProfileForm += \ From c212b2e8682b32aeae1696e6cb1e3554629c2229 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 6 Mar 2021 11:55:09 +0000 Subject: [PATCH 20/39] Indentation --- webapp_profile.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 5f0d868da..4df066eb2 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1211,7 +1211,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, '

' - skillsStr += '
\n' + skillsStr += '
\n' cssFilename = baseDir + '/epicyon-profile.css' if os.path.isfile(baseDir + '/epicyon.css'): @@ -1293,7 +1293,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' ' + \ translate['Broch mode'] + '
\n' - instanceStr += '' + instanceStr += ' \n' moderators = '' moderatorsFile = baseDir + '/accounts/moderators.txt' @@ -1309,7 +1309,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, translate['List of moderator nicknames'] + \ '..." style="height:200px" spellcheck="false">' + \ moderators + '' - moderatorsStr += '' + moderatorsStr += ' \n' editors = '' editorsFile = baseDir + '/accounts/editors.txt' @@ -1324,7 +1324,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' ' - editorsStr += '' + editorsStr += ' \n' themes = getThemesList(baseDir) themesDropdown = '
' @@ -1353,7 +1353,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' ' + \ translate['Remove the custom font'] + '
' - themesDropdown += '
' + themesDropdown += ' \n' themeName = getConfigParam(baseDir, 'theme') themesDropdown = \ themesDropdown.replace('