diff --git a/acceptreject.py b/acceptreject.py index 2a3c73123..4b056fb3d 100644 --- a/acceptreject.py +++ b/acceptreject.py @@ -17,7 +17,7 @@ from utils import domainPermitted from utils import followPerson from utils import hasObjectDict from utils import acctDir -from utils import hasGroupPath +from utils import hasGroupType def _createAcceptReject(baseDir: str, federationList: [], @@ -162,7 +162,9 @@ def _acceptFollow(baseDir: str, domain: str, messageJson: {}, return # does the url path indicate that this is a group actor - groupAccount = hasGroupPath(followedActor) + groupAccount = hasGroupType(baseDir, followedActor, None) + if debug: + print('Accepted follow is a group: ' + str(groupAccount)) if followPerson(baseDir, nickname, acceptedDomainFull, diff --git a/cache.py b/cache.py index 8d6291316..9ba0111fb 100644 --- a/cache.py +++ b/cache.py @@ -10,9 +10,11 @@ __module_group__ = "Core" import os import datetime from session import urlExists +from session import getJson from utils import loadJson from utils import saveJson from utils import getFileCaseInsensitive +from utils import getUserPaths def _removePersonFromCache(baseDir: str, personUrl: str, @@ -132,3 +134,52 @@ def getWebfingerFromCache(handle: str, cachedWebfingers: {}) -> {}: if cachedWebfingers.get(handle): return cachedWebfingers[handle] return None + + +def getPersonPubKey(baseDir: str, session, personUrl: str, + personCache: {}, debug: bool, + projectVersion: str, httpPrefix: str, + domain: str, onionDomain: str) -> str: + if not personUrl: + return None + personUrl = personUrl.replace('#main-key', '') + usersPaths = getUserPaths() + for possibleUsersPath in usersPaths: + if personUrl.endswith(possibleUsersPath + 'inbox'): + if debug: + print('DEBUG: Obtaining public key for shared inbox') + personUrl = \ + personUrl.replace(possibleUsersPath + 'inbox', '/inbox') + break + personJson = \ + getPersonFromCache(baseDir, personUrl, personCache, True) + if not personJson: + if debug: + print('DEBUG: Obtaining public key for ' + personUrl) + personDomain = domain + if onionDomain: + if '.onion/' in personUrl: + personDomain = onionDomain + profileStr = 'https://www.w3.org/ns/activitystreams' + asHeader = { + 'Accept': 'application/activity+json; profile="' + profileStr + '"' + } + personJson = \ + getJson(session, personUrl, asHeader, None, debug, + projectVersion, httpPrefix, personDomain) + if not personJson: + return None + pubKey = None + if personJson.get('publicKey'): + if personJson['publicKey'].get('publicKeyPem'): + pubKey = personJson['publicKey']['publicKeyPem'] + else: + if personJson.get('publicKeyPem'): + pubKey = personJson['publicKeyPem'] + + if not pubKey: + if debug: + print('DEBUG: Public key not found for ' + personUrl) + + storePersonInCache(baseDir, personUrl, personJson, personCache, True) + return pubKey diff --git a/daemon.py b/daemon.py index 7ba6cb06a..37582e1e3 100644 --- a/daemon.py +++ b/daemon.py @@ -92,7 +92,6 @@ from inbox import runInboxQueue from inbox import runInboxQueueWatchdog from inbox import savePostToInboxQueue from inbox import populateReplies -from inbox import getPersonPubKey from follow import isFollowingActor from follow import getFollowingFeed from follow import sendFollowRequest @@ -273,7 +272,7 @@ from utils import isSuspended from utils import dangerousMarkup from utils import refreshNewswire from utils import isImageFile -from utils import hasGroupPath +from utils import hasGroupType from manualapprove import manualDenyFollowRequest from manualapprove import manualApproveFollowRequest from announce import createAnnounce @@ -286,6 +285,7 @@ from media import processMetaData from cache import checkForChangedActor from cache import storePersonInCache from cache import getPersonFromCache +from cache import getPersonPubKey from httpsig import verifyPostHeaders from theme import importTheme from theme import exportTheme @@ -2569,7 +2569,9 @@ class PubServer(BaseHTTPRequestHandler): } pathUsersSection = path.split('/users/')[1] self.postToNickname = pathUsersSection.split('/')[0] - groupAccount = hasGroupPath(followingActor) + groupAccount = hasGroupType(self.server.baseDir, + followingActor, + self.server.personCache) unfollowAccount(self.server.baseDir, self.postToNickname, self.server.domain, followingNickname, followingDomainFull, diff --git a/follow.py b/follow.py index 5039d2df6..c391f09c7 100644 --- a/follow.py +++ b/follow.py @@ -28,13 +28,14 @@ from utils import saveJson from utils import isAccountDir from utils import getUserPaths from utils import acctDir -from utils import hasGroupPath +from utils import hasGroupType from acceptreject import createAccept from acceptreject import createReject from webfinger import webfingerHandle from auth import createBasicAuthHeader from session import getJson from session import postJson +from cache import getPersonPubKey def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None: @@ -626,7 +627,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, cachedWebfingers: {}, personCache: {}, messageJson: {}, federationList: [], debug: bool, projectVersion: str, - maxFollowers: int) -> bool: + maxFollowers: int, onionDomain: str) -> bool: """Receives a follow request within the POST section of HTTPServer """ if not messageJson['type'].startswith('Follow'): @@ -740,22 +741,34 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, else: print('Follow request does not require approval') # update the followers - if os.path.isdir(baseDir + '/accounts/' + - nicknameToFollow + '@' + domainToFollow): - followersFilename = \ - baseDir + '/accounts/' + \ - nicknameToFollow + '@' + domainToFollow + '/followers.txt' + accountToBeFollowed = \ + acctDir(baseDir, nicknameToFollow, domainToFollow) + if os.path.isdir(accountToBeFollowed): + followersFilename = accountToBeFollowed + '/followers.txt' # for actors which don't follow the mastodon # /users/ path convention store the full actor if '/users/' not in messageJson['actor']: approveHandle = messageJson['actor'] + # Get the actor for the follower and add it to the cache. + # Getting their public key has the same result + if debug: + print('Obtaining the following actor: ' + messageJson['actor']) + if not getPersonPubKey(baseDir, session, messageJson['actor'], + personCache, debug, projectVersion, + httpPrefix, domainToFollow, onionDomain): + if debug: + print('Unable to obtain following actor: ' + + messageJson['actor']) + print('Updating followers file: ' + followersFilename + ' adding ' + approveHandle) if os.path.isfile(followersFilename): if approveHandle not in open(followersFilename).read(): - groupAccount = hasGroupPath(messageJson['object']) + groupAccount = \ + hasGroupType(baseDir, messageJson['object'], + personCache) try: with open(followersFilename, 'r+') as followersFile: content = followersFile.read() @@ -925,7 +938,7 @@ def sendFollowRequest(session, baseDir: str, if followNickname: followedId = followedActor followHandle = followNickname + '@' + requestDomain - groupAccount = hasGroupPath(followedActor) + groupAccount = hasGroupType(baseDir, followedActor, personCache) if groupAccount: followHandle = '!' + followHandle else: @@ -1435,7 +1448,7 @@ def outboxUndoFollow(baseDir: str, messageJson: {}, debug: bool) -> None: getDomainFromActor(messageJson['object']['object']) domainFollowingFull = getFullDomain(domainFollowing, portFollowing) - groupAccount = hasGroupPath(messageJson['object']['object']) + groupAccount = hasGroupType(baseDir, messageJson['object']['object'], None) if unfollowAccount(baseDir, nicknameFollower, domainFollowerFull, nicknameFollowing, domainFollowingFull, debug, groupAccount): diff --git a/inbox.py b/inbox.py index b3628e512..4d697b49b 100644 --- a/inbox.py +++ b/inbox.py @@ -45,19 +45,18 @@ from utils import loadJson from utils import saveJson from utils import updateLikesCollection from utils import undoLikesCollectionEntry -from utils import hasGroupPath +from utils import hasGroupType from categories import getHashtagCategories from categories import setHashtagCategory from httpsig import verifyPostHeaders from session import createSession -from session import getJson from follow import isFollowingActor from follow import receiveFollowRequest from follow import getFollowersOfActor from follow import unfollowerOfAccount from pprint import pprint -from cache import getPersonFromCache from cache import storePersonInCache +from cache import getPersonPubKey from acceptreject import receiveAcceptReject from bookmarks import updateBookmarksCollection from bookmarks import undoBookmarksCollectionEntry @@ -234,55 +233,6 @@ def validInboxFilenames(baseDir: str, nickname: str, domain: str, return True -def getPersonPubKey(baseDir: str, session, personUrl: str, - personCache: {}, debug: bool, - projectVersion: str, httpPrefix: str, - domain: str, onionDomain: str) -> str: - if not personUrl: - return None - personUrl = personUrl.replace('#main-key', '') - usersPaths = getUserPaths() - for possibleUsersPath in usersPaths: - if personUrl.endswith(possibleUsersPath + 'inbox'): - if debug: - print('DEBUG: Obtaining public key for shared inbox') - personUrl = \ - personUrl.replace(possibleUsersPath + 'inbox', '/inbox') - break - personJson = \ - getPersonFromCache(baseDir, personUrl, personCache, True) - if not personJson: - if debug: - print('DEBUG: Obtaining public key for ' + personUrl) - personDomain = domain - if onionDomain: - if '.onion/' in personUrl: - personDomain = onionDomain - profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/activity+json; profile="' + profileStr + '"' - } - personJson = \ - getJson(session, personUrl, asHeader, None, debug, - projectVersion, httpPrefix, personDomain) - if not personJson: - return None - pubKey = None - if personJson.get('publicKey'): - if personJson['publicKey'].get('publicKeyPem'): - pubKey = personJson['publicKey']['publicKeyPem'] - else: - if personJson.get('publicKeyPem'): - pubKey = personJson['publicKeyPem'] - - if not pubKey: - if debug: - print('DEBUG: Public key not found for ' + personUrl) - - storePersonInCache(baseDir, personUrl, personJson, personCache, True) - return pubKey - - def inboxMessageHasParams(messageJson: {}) -> bool: """Checks whether an incoming message contains expected parameters """ @@ -676,7 +626,7 @@ def _receiveUndoFollow(session, baseDir: str, httpPrefix: str, getDomainFromActor(messageJson['object']['object']) domainFollowingFull = getFullDomain(domainFollowing, portFollowing) - groupAccount = hasGroupPath(messageJson['object']['actor']) + groupAccount = hasGroupType(baseDir, messageJson['object']['actor'], None) if unfollowerOfAccount(baseDir, nicknameFollowing, domainFollowingFull, nicknameFollower, domainFollowerFull, @@ -3148,7 +3098,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queueJson['post'], federationList, debug, projectVersion, - maxFollowers): + maxFollowers, onionDomain): if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: diff --git a/migrate.py b/migrate.py index 91f323e8b..a093cc8fe 100644 --- a/migrate.py +++ b/migrate.py @@ -12,7 +12,7 @@ from utils import isAccountDir from utils import getNicknameFromActor from utils import getDomainFromActor from utils import acctDir -from utils import hasGroupPath +from utils import hasGroupType from webfinger import webfingerHandle from blocking import isBlocked from posts import getUserUrl @@ -103,7 +103,7 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, if movedToPort: if movedToPort != 80 and movedToPort != 443: movedToDomainFull = movedToDomain + ':' + str(movedToPort) - groupAccount = hasGroupPath(movedToUrl) + groupAccount = hasGroupType(baseDir, movedToUrl, None) if isBlocked(baseDir, nickname, domain, movedToNickname, movedToDomain): # someone that you follow has moved to a blocked domain diff --git a/tests.py b/tests.py index c3bed878d..51651a90c 100644 --- a/tests.py +++ b/tests.py @@ -39,6 +39,7 @@ from follow import clearFollowers from follow import sendFollowRequestViaServer from follow import sendUnfollowRequestViaServer from siteactive import siteIsActive +from utils import isGroupActor from utils import dateStringToSeconds from utils import dateSecondsToString from utils import validPassword @@ -1261,6 +1262,7 @@ def testFollowBetweenServers(): assert 'bob@' + bobDomain in open(aliceDir + '/accounts/alice@' + aliceDomain + '/followingCalendar.txt').read() + assert not isGroupActor(aliceDir, bobActor, alicePersonCache) print('\n\n*********************************************************') print('Alice sends a message to Bob') @@ -1319,7 +1321,10 @@ def testGroupFollow(): print('Testing following of a group') global testServerAliceRunning + global testServerGroupRunning + systemLanguage = 'en' testServerAliceRunning = False + testServerGroupRunning = False # systemLanguage = 'en' httpPrefix = 'http' @@ -1399,17 +1404,18 @@ def testGroupFollow(): print('Alice sends a follow request to the test group') os.chdir(aliceDir) sessionAlice = createSession(proxyType) - # inReplyTo = None - # inReplyToAtomUri = None - # subject = None + inReplyTo = None + inReplyToAtomUri = None + subject = None alicePostLog = [] - # followersOnly = False - # saveToFile = True + followersOnly = False + saveToFile = True clientToServer = False - # ccUrl = None + ccUrl = None alicePersonCache = {} aliceCachedWebfingers = {} alicePostLog = [] + # aliceActor = httpPrefix + '://' + aliceAddress + '/users/alice' testgroupActor = httpPrefix + '://' + testgroupAddress + '/users/testgroup' sendResult = \ sendFollowRequest(sessionAlice, aliceDir, @@ -1422,32 +1428,88 @@ def testGroupFollow(): True, __version__) print('sendResult: ' + str(sendResult)) + aliceFollowingFilename = \ + aliceDir + '/accounts/alice@' + aliceDomain + '/following.txt' + aliceFollowingCalendarFilename = \ + aliceDir + '/accounts/alice@' + aliceDomain + \ + '/followingCalendar.txt' + testgroupFollowersFilename = \ + testgroupDir + '/accounts/testgroup@' + testgroupDomain + \ + '/followers.txt' + for t in range(16): - if os.path.isfile(testgroupDir + '/accounts/testgroup@' + - testgroupDomain + '/followers.txt'): - if os.path.isfile(aliceDir + '/accounts/alice@' + - aliceDomain + '/following.txt'): - if os.path.isfile(aliceDir + '/accounts/alice@' + - aliceDomain + '/followingCalendar.txt'): + if os.path.isfile(testgroupFollowersFilename): + if os.path.isfile(aliceFollowingFilename): + if os.path.isfile(aliceFollowingCalendarFilename): break time.sleep(1) assert validInbox(testgroupDir, 'testgroup', testgroupDomain) assert validInboxFilenames(testgroupDir, 'testgroup', testgroupDomain, aliceDomain, alicePort) - assert 'alice@' + aliceDomain in open(testgroupDir + - '/accounts/testgroup@' + - testgroupDomain + - '/followers.txt').read() - assert 'testgroup@' + testgroupDomain in open(aliceDir + - '/accounts/alice@' + - aliceDomain + - '/following.txt').read() + assert 'alice@' + aliceDomain in open(testgroupFollowersFilename).read() + + testgroupWebfingerFilename = \ + testgroupDir + '/wfendpoints/testgroup@' + \ + testgroupDomain + ':' + str(testgroupPort) + '.json' + assert os.path.isfile(testgroupWebfingerFilename) + assert 'group:testgroup@' in open(testgroupWebfingerFilename).read() + print('group: exists within the webfinger endpoint for testgroup') + testgroupHandle = 'testgroup@' + testgroupDomain - assert testgroupHandle in open(aliceDir + - '/accounts/alice@' + - aliceDomain + - '/followingCalendar.txt').read() + followingStr = '' + with open(aliceFollowingFilename, 'r') as fp: + followingStr = fp.read() + print('Alice following.txt:\n\n' + followingStr) + if '!testgroup' not in followingStr: + print('Alice following.txt does not contain !testgroup@' + + testgroupDomain + ':' + str(testgroupPort)) + assert isGroupActor(aliceDir, testgroupActor, alicePersonCache) + assert '!testgroup' in followingStr + assert testgroupHandle in open(aliceFollowingFilename).read() + assert testgroupHandle in open(aliceFollowingCalendarFilename).read() + print('Alice follows the test group') + + print('\n\n*********************************************************') + print('Alice posts to the test group') + alicePostLog = [] + alicePersonCache = {} + aliceCachedWebfingers = {} + alicePostLog = [] + isArticle = False + city = 'London, England' + sendResult = \ + sendPost(__version__, + sessionAlice, aliceDir, 'alice', aliceDomain, alicePort, + 'testgroup', testgroupDomain, testgroupPort, ccUrl, + httpPrefix, "Alice group message", followersOnly, + saveToFile, clientToServer, True, + None, None, None, city, federationList, + aliceSendThreads, alicePostLog, aliceCachedWebfingers, + alicePersonCache, isArticle, systemLanguage, inReplyTo, + inReplyToAtomUri, subject) + print('sendResult: ' + str(sendResult)) + + queuePath = \ + testgroupDir + '/accounts/testgroup@' + testgroupDomain + '/queue' + inboxPath = \ + testgroupDir + '/accounts/testgroup@' + testgroupDomain + '/inbox' + aliceMessageArrived = False + startPosts = len([name for name in os.listdir(inboxPath) + if os.path.isfile(os.path.join(inboxPath, name))]) + for i in range(20): + time.sleep(1) + if os.path.isdir(inboxPath): + currPosts = \ + len([name for name in os.listdir(inboxPath) + if os.path.isfile(os.path.join(inboxPath, name))]) + if currPosts > startPosts: + aliceMessageArrived = True + print('Alice post sent to test group!') + break + + assert aliceMessageArrived is True + print('Post from Alice to test group succeeded') # stop the servers thrAlice.kill() @@ -1465,6 +1527,7 @@ def testGroupFollow(): os.chdir(baseDir) shutil.rmtree(baseDir + '/.tests') + print('Testing following of a group is complete') def _testFollowersOfPerson(): diff --git a/utils.py b/utils.py index fe9726d1c..e5a7fad7b 100644 --- a/utils.py +++ b/utils.py @@ -1081,6 +1081,9 @@ def followPerson(baseDir: str, nickname: str, domain: str, else: handleToFollow = followNickname + '@' + followDomain + if groupAccount: + handleToFollow = '!' + handleToFollow + # was this person previously unfollowed? unfollowedFilename = baseDir + '/accounts/' + handle + '/unfollowed.txt' if os.path.isfile(unfollowedFilename): @@ -1098,6 +1101,8 @@ def followPerson(baseDir: str, nickname: str, domain: str, if not os.path.isdir(baseDir + '/accounts'): os.mkdir(baseDir + '/accounts') handleToFollow = followNickname + '@' + followDomain + if groupAccount: + handleToFollow = '!' + handleToFollow filename = baseDir + '/accounts/' + handle + '/' + followFile if os.path.isfile(filename): if handleToFollow in open(filename).read(): @@ -2682,13 +2687,33 @@ def dateSecondsToString(dateSec: int) -> str: return thisDate.strftime("%Y-%m-%dT%H:%M:%SZ") -def hasGroupPath(actor: str) -> bool: - """Does the given actor url contain a path which indicates - that it belongs to a group? - e.g. https://lemmy/c/groupname +def hasGroupType(baseDir: str, actor: str, personCache: {}) -> bool: + """Does the given actor url have a group type? """ + # does the actor path clearly indicate that this is a group? + # eg. https://lemmy/c/groupname groupPaths = getGroupPaths() for grpPath in groupPaths: if grpPath in actor: return True + # is there a cached actor which can be examined for Group type? + return isGroupActor(baseDir, actor, personCache) + + +def isGroupActor(baseDir: str, actor: str, personCache: {}) -> bool: + """Is the given actor a group? + """ + if personCache: + if personCache.get(actor): + if personCache[actor].get('actor'): + if personCache[actor]['actor'].get('type'): + if personCache[actor]['actor']['type'] == 'Group': + return True + return False + cachedActorFilename = \ + baseDir + '/cache/actors/' + (actor.replace('/', '#')) + '.json' + if not os.path.isfile(cachedActorFilename): + return False + if '"type": "Group"' in open(cachedActorFilename).read(): + return True return False