From b870a6954528ef18ea93259349cce5fb534f7b63 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 4 Jul 2021 23:46:53 +0100 Subject: [PATCH 1/7] Tidying --- person.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/person.py b/person.py index 3e670e381..c6fd12e49 100644 --- a/person.py +++ b/person.py @@ -1302,11 +1302,11 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool, if not personUrl: personUrl = getUserUrl(wfRequest, 0, debug) if nickname == domain: - personUrl = personUrl.replace('/users/', '/actor/') - personUrl = personUrl.replace('/accounts/', '/actor/') - personUrl = personUrl.replace('/channel/', '/actor/') - personUrl = personUrl.replace('/profile/', '/actor/') - personUrl = personUrl.replace('/u/', '/actor/') + paths = ( + '/users/', '/accounts/', '/channel/', '/profile/', '/u/' + ) + for userPath in paths: + personUrl = personUrl.replace(userPath, '/actor/') if not personUrl: # try single user instance personUrl = httpPrefix + '://' + domain + '/' + nickname From 0335e524581a43efd63dee73c348d307e03d3140 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 4 Jul 2021 23:58:01 +0100 Subject: [PATCH 2/7] Function to return user paths --- follow.py | 3 ++- manualapprove.py | 3 ++- person.py | 9 +++------ utils.py | 10 ++++++++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/follow.py b/follow.py index 0ef1c5ce6..76129fe55 100644 --- a/follow.py +++ b/follow.py @@ -26,6 +26,7 @@ from posts import getPersonBox from utils import loadJson from utils import saveJson from utils import isAccountDir +from utils import getUserPaths from acceptreject import createAccept from acceptreject import createReject from webfinger import webfingerHandle @@ -221,7 +222,7 @@ def isFollowerOfPerson(baseDir: str, nickname: str, domain: str, if handle in followersStr: alreadyFollowing = True else: - paths = ('/profile/', '/channel/', '/accounts/', '/u/') + paths = getUserPaths() for userPath in paths: url = '://' + followerDomain + userPath + followerNickname if url in followersStr: diff --git a/manualapprove.py b/manualapprove.py index c915683ef..968b016e3 100644 --- a/manualapprove.py +++ b/manualapprove.py @@ -14,6 +14,7 @@ from follow import removeFromFollowRequests from utils import loadJson from utils import removeDomainPort from utils import getPortFromDomain +from utils import getUserPaths def manualDenyFollowRequest(session, baseDir: str, @@ -111,7 +112,7 @@ def manualApproveFollowRequest(session, baseDir: str, reqNick = approveHandle.split('@')[0] reqDomain = approveHandle.split('@')[1].strip() reqPrefix = httpPrefix + '://' + reqDomain - paths = ('/profile/', '/channel/', '/accounts/', '/u/') + paths = getUserPaths() for userPath in paths: if reqPrefix + userPath + reqNick in approveFollowsStr: exists = True diff --git a/person.py b/person.py index c6fd12e49..1f9d3cd12 100644 --- a/person.py +++ b/person.py @@ -50,6 +50,7 @@ from utils import getProtocolPrefixes from utils import hasUsersPath from utils import getImageExtensions from utils import isImageFile +from utils import getUserPaths from session import createSession from session import getJson from webfinger import webfingerHandle @@ -1211,9 +1212,7 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool, for prefix in prefixes: handle = handle.replace(prefix, '') handle = handle.replace('/@', '/users/') - paths = ( - '/users/', '/profile/', '/channel/', '/accounts/', '/u/' - ) + paths = getUserPaths() userPathFound = False for userPath in paths: if userPath in handle: @@ -1302,9 +1301,7 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool, if not personUrl: personUrl = getUserUrl(wfRequest, 0, debug) if nickname == domain: - paths = ( - '/users/', '/accounts/', '/channel/', '/profile/', '/u/' - ) + paths = getUserPaths() for userPath in paths: personUrl = personUrl.replace(userPath, '/actor/') if not personUrl: diff --git a/utils.py b/utils.py index 70b036cac..5aafb559c 100644 --- a/utils.py +++ b/utils.py @@ -842,7 +842,7 @@ def getNicknameFromActor(actor: str) -> str: """ if actor.startswith('@'): actor = actor[1:] - usersPaths = ('/users/', '/profile/', '/channel/', '/accounts/', '/u/') + usersPaths = getUserPaths() for possiblePath in usersPaths: if possiblePath in actor: nickStr = actor.split(possiblePath)[1].replace('@', '') @@ -872,6 +872,12 @@ def getNicknameFromActor(actor: str) -> str: return None +def getUserPaths() -> []: + """Returns possible user paths + """ + return ('/users/', '/profile/', '/accounts/', '/channel/', '/u/') + + def getDomainFromActor(actor: str) -> (str, int): """Returns the domain name from an actor url """ @@ -879,7 +885,7 @@ def getDomainFromActor(actor: str) -> (str, int): actor = actor[1:] port = None prefixes = getProtocolPrefixes() - usersPaths = ('/users/', '/profile/', '/accounts/', '/channel/', '/u/') + usersPaths = getUserPaths() for possiblePath in usersPaths: if possiblePath in actor: domain = actor.split(possiblePath)[0] From 021556ccee43f224ab4fb27207b5ac26c1b11c60 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 5 Jul 2021 10:24:29 +0100 Subject: [PATCH 3/7] Tidying of post deletion --- utils.py | 235 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 132 insertions(+), 103 deletions(-) diff --git a/utils.py b/utils.py index 5aafb559c..4c7adda9c 100644 --- a/utils.py +++ b/utils.py @@ -1228,125 +1228,154 @@ def _isReplyToBlogPost(baseDir: str, nickname: str, domain: str, return False +def _deletePostRemoveReplies(baseDir: str, nickname: str, domain: str, + httpPrefix: str, postFilename: str, + recentPostsCache: {}, debug: bool) -> None: + """Removes replies when deleting a post + """ + repliesFilename = postFilename.replace('.json', '.replies') + if not os.path.isfile(repliesFilename): + return + if debug: + print('DEBUG: removing replies to ' + postFilename) + with open(repliesFilename, 'r') as f: + for replyId in f: + replyFile = locatePost(baseDir, nickname, domain, replyId) + if not replyFile: + continue + if os.path.isfile(replyFile): + deletePost(baseDir, httpPrefix, + nickname, domain, replyFile, debug, + recentPostsCache) + # remove the replies file + os.remove(repliesFilename) + + +def _isBookmarked(baseDir: str, nickname: str, domain: str, + postFilename: str) -> bool: + """Returns True if the given post is bookmarked + """ + bookmarksIndexFilename = \ + baseDir + '/accounts/' + nickname + '@' + domain + \ + '/bookmarks.index' + if os.path.isfile(bookmarksIndexFilename): + bookmarkIndex = postFilename.split('/')[-1] + '\n' + if bookmarkIndex in open(bookmarksIndexFilename).read(): + return True + return False + + def deletePost(baseDir: str, httpPrefix: str, nickname: str, domain: str, postFilename: str, debug: bool, recentPostsCache: {}) -> None: """Recursively deletes a post and its replies and attachments """ postJsonObject = loadJson(postFilename, 1) - if postJsonObject: - # don't allow deletion of bookmarked posts - bookmarksIndexFilename = \ - baseDir + '/accounts/' + nickname + '@' + domain + \ - '/bookmarks.index' - if os.path.isfile(bookmarksIndexFilename): - bookmarkIndex = postFilename.split('/')[-1] + '\n' - if bookmarkIndex in open(bookmarksIndexFilename).read(): - return + if not postJsonObject: + # remove any replies + _deletePostRemoveReplies(baseDir, nickname, domain, + httpPrefix, postFilename, + recentPostsCache, debug) + # finally, remove the post itself + os.remove(postFilename) + return - # don't remove replies to blog posts - if _isReplyToBlogPost(baseDir, nickname, domain, - postJsonObject): - return + # don't allow deletion of bookmarked posts + if _isBookmarked(baseDir, nickname, domain, postFilename): + return - # remove from recent posts cache in memory - if recentPostsCache: - postId = \ - removeIdEnding(postJsonObject['id']).replace('/', '#') - if recentPostsCache.get('index'): - if postId in recentPostsCache['index']: - recentPostsCache['index'].remove(postId) - if recentPostsCache.get('json'): - if recentPostsCache['json'].get(postId): - del recentPostsCache['json'][postId] - if recentPostsCache.get('html'): - if recentPostsCache['html'].get(postId): - del recentPostsCache['html'][postId] + # don't remove replies to blog posts + if _isReplyToBlogPost(baseDir, nickname, domain, + postJsonObject): + return - # remove any attachment - _removeAttachment(baseDir, httpPrefix, domain, postJsonObject) + # remove from recent posts cache in memory + if recentPostsCache: + postId = \ + removeIdEnding(postJsonObject['id']).replace('/', '#') + if recentPostsCache.get('index'): + if postId in recentPostsCache['index']: + recentPostsCache['index'].remove(postId) + if recentPostsCache.get('json'): + if recentPostsCache['json'].get(postId): + del recentPostsCache['json'][postId] + if recentPostsCache.get('html'): + if recentPostsCache['html'].get(postId): + del recentPostsCache['html'][postId] - extensions = ('votes', 'arrived', 'muted', 'tts', 'reject') - for ext in extensions: - extFilename = postFilename + '.' + ext - if os.path.isfile(extFilename): - os.remove(extFilename) + # remove any attachment + _removeAttachment(baseDir, httpPrefix, domain, postJsonObject) - # remove cached html version of the post - cachedPostFilename = \ - getCachedPostFilename(baseDir, nickname, domain, postJsonObject) - if cachedPostFilename: - if os.path.isfile(cachedPostFilename): - os.remove(cachedPostFilename) - # removePostFromCache(postJsonObject,recentPostsCache) + extensions = ('votes', 'arrived', 'muted', 'tts', 'reject') + for ext in extensions: + extFilename = postFilename + '.' + ext + if os.path.isfile(extFilename): + os.remove(extFilename) - hasObject = False - if postJsonObject.get('object'): - hasObject = True + # remove cached html version of the post + cachedPostFilename = \ + getCachedPostFilename(baseDir, nickname, domain, postJsonObject) + if cachedPostFilename: + if os.path.isfile(cachedPostFilename): + os.remove(cachedPostFilename) + # removePostFromCache(postJsonObject,recentPostsCache) - # remove from moderation index file - if hasObject: - if isinstance(postJsonObject['object'], dict): - if postJsonObject['object'].get('moderationStatus'): - if postJsonObject.get('id'): - postId = removeIdEnding(postJsonObject['id']) - removeModerationPostFromIndex(baseDir, postId, debug) + hasObject = False + if postJsonObject.get('object'): + hasObject = True - # remove any hashtags index entries - removeHashtagIndex = False - if hasObject: - if hasObject and isinstance(postJsonObject['object'], dict): - if postJsonObject['object'].get('content'): - if '#' in postJsonObject['object']['content']: - removeHashtagIndex = True - if removeHashtagIndex: - if postJsonObject['object'].get('id') and \ - postJsonObject['object'].get('tag'): - # get the id of the post - postId = removeIdEnding(postJsonObject['object']['id']) - for tag in postJsonObject['object']['tag']: - if tag['type'] != 'Hashtag': - continue - if not tag.get('name'): - continue - # find the index file for this tag - tagIndexFilename = \ - baseDir + '/tags/' + tag['name'][1:] + '.txt' - if not os.path.isfile(tagIndexFilename): - continue - # remove postId from the tag index file - lines = None - with open(tagIndexFilename, "r") as f: - lines = f.readlines() - if lines: - newlines = '' - for fileLine in lines: - if postId in fileLine: - continue - newlines += fileLine - if not newlines.strip(): - # if there are no lines then remove the - # hashtag file - os.remove(tagIndexFilename) - else: - with open(tagIndexFilename, "w+") as f: - f.write(newlines) + # remove from moderation index file + if hasObject: + if isinstance(postJsonObject['object'], dict): + if postJsonObject['object'].get('moderationStatus'): + if postJsonObject.get('id'): + postId = removeIdEnding(postJsonObject['id']) + removeModerationPostFromIndex(baseDir, postId, debug) + + # remove any hashtags index entries + removeHashtagIndex = False + if hasObject: + if hasObject and isinstance(postJsonObject['object'], dict): + if postJsonObject['object'].get('content'): + if '#' in postJsonObject['object']['content']: + removeHashtagIndex = True + if removeHashtagIndex: + if postJsonObject['object'].get('id') and \ + postJsonObject['object'].get('tag'): + # get the id of the post + postId = removeIdEnding(postJsonObject['object']['id']) + for tag in postJsonObject['object']['tag']: + if tag['type'] != 'Hashtag': + continue + if not tag.get('name'): + continue + # find the index file for this tag + tagIndexFilename = \ + baseDir + '/tags/' + tag['name'][1:] + '.txt' + if not os.path.isfile(tagIndexFilename): + continue + # remove postId from the tag index file + lines = None + with open(tagIndexFilename, "r") as f: + lines = f.readlines() + if lines: + newlines = '' + for fileLine in lines: + if postId in fileLine: + continue + newlines += fileLine + if not newlines.strip(): + # if there are no lines then remove the + # hashtag file + os.remove(tagIndexFilename) + else: + with open(tagIndexFilename, "w+") as f: + f.write(newlines) # remove any replies - repliesFilename = postFilename.replace('.json', '.replies') - if os.path.isfile(repliesFilename): - if debug: - print('DEBUG: removing replies to ' + postFilename) - with open(repliesFilename, 'r') as f: - for replyId in f: - replyFile = locatePost(baseDir, nickname, domain, replyId) - if replyFile: - if os.path.isfile(replyFile): - deletePost(baseDir, httpPrefix, - nickname, domain, replyFile, debug, - recentPostsCache) - # remove the replies file - os.remove(repliesFilename) + _deletePostRemoveReplies(baseDir, nickname, domain, + httpPrefix, postFilename, + recentPostsCache, debug) # finally, remove the post itself os.remove(postFilename) From 4ba64797ef5111af4f3b1a98ac65e1b3820b9de1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 5 Jul 2021 10:45:55 +0100 Subject: [PATCH 4/7] Tidying of post deletion --- utils.py | 137 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 55 deletions(-) diff --git a/utils.py b/utils.py index 4c7adda9c..cc276af35 100644 --- a/utils.py +++ b/utils.py @@ -1265,6 +1265,85 @@ def _isBookmarked(baseDir: str, nickname: str, domain: str, return False +def _deleteFromRecentPosts(postJsonObject: {}, recentPostsCache: {}) -> None: + """Remove the given post from the recent posts cache + """ + if not recentPostsCache: + return + + postId = \ + removeIdEnding(postJsonObject['id']).replace('/', '#') + + if recentPostsCache.get('index'): + if postId in recentPostsCache['index']: + recentPostsCache['index'].remove(postId) + + if recentPostsCache.get('json'): + if recentPostsCache['json'].get(postId): + del recentPostsCache['json'][postId] + + if recentPostsCache.get('html'): + if recentPostsCache['html'].get(postId): + del recentPostsCache['html'][postId] + + +def _deleteCachedHtml(baseDir: str, nickname: str, domain: str, + postJsonObject: {}): + """Removes cached html file for the given post + """ + cachedPostFilename = \ + getCachedPostFilename(baseDir, nickname, domain, postJsonObject) + if cachedPostFilename: + if os.path.isfile(cachedPostFilename): + os.remove(cachedPostFilename) + + +def _deleteHashtagsOnPost(baseDir: str, postJsonObject: {}) -> None: + """Removes hashtags when a post is deleted + """ + removeHashtagIndex = False + if isinstance(postJsonObject['object'], dict): + if postJsonObject['object'].get('content'): + if '#' in postJsonObject['object']['content']: + removeHashtagIndex = True + + if not removeHashtagIndex: + return + + if not postJsonObject['object'].get('id') or \ + not postJsonObject['object'].get('tag'): + return + + # get the id of the post + postId = removeIdEnding(postJsonObject['object']['id']) + for tag in postJsonObject['object']['tag']: + if tag['type'] != 'Hashtag': + continue + if not tag.get('name'): + continue + # find the index file for this tag + tagIndexFilename = baseDir + '/tags/' + tag['name'][1:] + '.txt' + if not os.path.isfile(tagIndexFilename): + continue + # remove postId from the tag index file + lines = None + with open(tagIndexFilename, "r") as f: + lines = f.readlines() + if not lines: + continue + newlines = '' + for fileLine in lines: + if postId in fileLine: + continue + newlines += fileLine + if not newlines.strip(): + # if there are no lines then remove the hashtag file + os.remove(tagIndexFilename) + else: + with open(tagIndexFilename, "w+") as f: + f.write(newlines) + + def deletePost(baseDir: str, httpPrefix: str, nickname: str, domain: str, postFilename: str, debug: bool, recentPostsCache: {}) -> None: @@ -1290,18 +1369,7 @@ def deletePost(baseDir: str, httpPrefix: str, return # remove from recent posts cache in memory - if recentPostsCache: - postId = \ - removeIdEnding(postJsonObject['id']).replace('/', '#') - if recentPostsCache.get('index'): - if postId in recentPostsCache['index']: - recentPostsCache['index'].remove(postId) - if recentPostsCache.get('json'): - if recentPostsCache['json'].get(postId): - del recentPostsCache['json'][postId] - if recentPostsCache.get('html'): - if recentPostsCache['html'].get(postId): - del recentPostsCache['html'][postId] + _deleteFromRecentPosts(postJsonObject, recentPostsCache) # remove any attachment _removeAttachment(baseDir, httpPrefix, domain, postJsonObject) @@ -1313,12 +1381,7 @@ def deletePost(baseDir: str, httpPrefix: str, os.remove(extFilename) # remove cached html version of the post - cachedPostFilename = \ - getCachedPostFilename(baseDir, nickname, domain, postJsonObject) - if cachedPostFilename: - if os.path.isfile(cachedPostFilename): - os.remove(cachedPostFilename) - # removePostFromCache(postJsonObject,recentPostsCache) + _deleteCachedHtml(baseDir, nickname, domain, postJsonObject) hasObject = False if postJsonObject.get('object'): @@ -1333,44 +1396,8 @@ def deletePost(baseDir: str, httpPrefix: str, removeModerationPostFromIndex(baseDir, postId, debug) # remove any hashtags index entries - removeHashtagIndex = False if hasObject: - if hasObject and isinstance(postJsonObject['object'], dict): - if postJsonObject['object'].get('content'): - if '#' in postJsonObject['object']['content']: - removeHashtagIndex = True - if removeHashtagIndex: - if postJsonObject['object'].get('id') and \ - postJsonObject['object'].get('tag'): - # get the id of the post - postId = removeIdEnding(postJsonObject['object']['id']) - for tag in postJsonObject['object']['tag']: - if tag['type'] != 'Hashtag': - continue - if not tag.get('name'): - continue - # find the index file for this tag - tagIndexFilename = \ - baseDir + '/tags/' + tag['name'][1:] + '.txt' - if not os.path.isfile(tagIndexFilename): - continue - # remove postId from the tag index file - lines = None - with open(tagIndexFilename, "r") as f: - lines = f.readlines() - if lines: - newlines = '' - for fileLine in lines: - if postId in fileLine: - continue - newlines += fileLine - if not newlines.strip(): - # if there are no lines then remove the - # hashtag file - os.remove(tagIndexFilename) - else: - with open(tagIndexFilename, "w+") as f: - f.write(newlines) + _deleteHashtagsOnPost(baseDir, postJsonObject) # remove any replies _deletePostRemoveReplies(baseDir, nickname, domain, From 8ae7262e9a5952698575ec646338820f0a05fdd9 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 5 Jul 2021 10:51:07 +0100 Subject: [PATCH 5/7] Use function to check for dict --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index cc276af35..eb4bc2c65 100644 --- a/utils.py +++ b/utils.py @@ -1302,7 +1302,7 @@ def _deleteHashtagsOnPost(baseDir: str, postJsonObject: {}) -> None: """Removes hashtags when a post is deleted """ removeHashtagIndex = False - if isinstance(postJsonObject['object'], dict): + if hasObjectDict(postJsonObject): if postJsonObject['object'].get('content'): if '#' in postJsonObject['object']['content']: removeHashtagIndex = True From 3a509400a74a7597460ea50e7a1906b1e5d0218f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 5 Jul 2021 10:55:01 +0100 Subject: [PATCH 6/7] Comments --- utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.py b/utils.py index eb4bc2c65..d2f731618 100644 --- a/utils.py +++ b/utils.py @@ -1334,12 +1334,14 @@ def _deleteHashtagsOnPost(baseDir: str, postJsonObject: {}) -> None: newlines = '' for fileLine in lines: if postId in fileLine: + # skip over the deleted post continue newlines += fileLine if not newlines.strip(): # if there are no lines then remove the hashtag file os.remove(tagIndexFilename) else: + # write the new hashtag index without the given post in it with open(tagIndexFilename, "w+") as f: f.write(newlines) From 607f54eb5bcc751fa1c41a17ff73f4f36db5590e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 5 Jul 2021 11:09:11 +0100 Subject: [PATCH 7/7] Remove redundant function --- utils.py | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/utils.py b/utils.py index d2f731618..b8d6ae965 100644 --- a/utils.py +++ b/utils.py @@ -1265,14 +1265,24 @@ def _isBookmarked(baseDir: str, nickname: str, domain: str, return False -def _deleteFromRecentPosts(postJsonObject: {}, recentPostsCache: {}) -> None: - """Remove the given post from the recent posts cache +def removePostFromCache(postJsonObject: {}, recentPostsCache: {}) -> None: + """ if the post exists in the recent posts cache then remove it """ if not recentPostsCache: return - postId = \ - removeIdEnding(postJsonObject['id']).replace('/', '#') + if not postJsonObject.get('id'): + return + + if not recentPostsCache.get('index'): + return + + postId = postJsonObject['id'] + if '#' in postId: + postId = postId.split('#', 1)[0] + postId = removeIdEnding(postId).replace('/', '#') + if postId not in recentPostsCache['index']: + return if recentPostsCache.get('index'): if postId in recentPostsCache['index']: @@ -1371,7 +1381,7 @@ def deletePost(baseDir: str, httpPrefix: str, return # remove from recent posts cache in memory - _deleteFromRecentPosts(postJsonObject, recentPostsCache) + removePostFromCache(postJsonObject, recentPostsCache) # remove any attachment _removeAttachment(baseDir, httpPrefix, domain, postJsonObject) @@ -1607,29 +1617,6 @@ def getCachedPostFilename(baseDir: str, nickname: str, domain: str, return cachedPostFilename + '.html' -def removePostFromCache(postJsonObject: {}, recentPostsCache: {}): - """ if the post exists in the recent posts cache then remove it - """ - if not postJsonObject.get('id'): - return - - if not recentPostsCache.get('index'): - return - - postId = postJsonObject['id'] - if '#' in postId: - postId = postId.split('#', 1)[0] - postId = removeIdEnding(postId).replace('/', '#') - if postId not in recentPostsCache['index']: - return - - if recentPostsCache['json'].get(postId): - del recentPostsCache['json'][postId] - if recentPostsCache['html'].get(postId): - del recentPostsCache['html'][postId] - recentPostsCache['index'].remove(postId) - - def updateRecentPostsCache(recentPostsCache: {}, maxRecentPosts: int, postJsonObject: {}, htmlStr: str) -> None: """Store recent posts in memory so that they can be quickly recalled