Unit test for following a group

acceptreject.py

@@ -17,7 +17,7 @@ from utils import domainPermitted
 from utils import followPerson
 from utils import hasObjectDict
 from utils import acctDir
-from utils import hasGroupPath
+from utils import hasGroupType
 
 
 def _createAcceptReject(baseDir: str, federationList: [],
@@ -162,7 +162,9 @@ def _acceptFollow(baseDir: str, domain: str, messageJson: {},
             return
 
     # does the url path indicate that this is a group actor
-    groupAccount = hasGroupPath(followedActor)
+    groupAccount = hasGroupType(baseDir, followedActor, None)
+    if debug:
+        print('Accepted follow is a group: ' + str(groupAccount))
 
     if followPerson(baseDir,
                     nickname, acceptedDomainFull,

cache.py

@@ -10,9 +10,11 @@ __module_group__ = "Core"
 import os
 import datetime
 from session import urlExists
+from session import getJson
 from utils import loadJson
 from utils import saveJson
 from utils import getFileCaseInsensitive
+from utils import getUserPaths
 
 
 def _removePersonFromCache(baseDir: str, personUrl: str,
@@ -132,3 +134,52 @@ def getWebfingerFromCache(handle: str, cachedWebfingers: {}) -> {}:
     if cachedWebfingers.get(handle):
         return cachedWebfingers[handle]
     return None
+
+
+def getPersonPubKey(baseDir: str, session, personUrl: str,
+                    personCache: {}, debug: bool,
+                    projectVersion: str, httpPrefix: str,
+                    domain: str, onionDomain: str) -> str:
+    if not personUrl:
+        return None
+    personUrl = personUrl.replace('#main-key', '')
+    usersPaths = getUserPaths()
+    for possibleUsersPath in usersPaths:
+        if personUrl.endswith(possibleUsersPath + 'inbox'):
+            if debug:
+                print('DEBUG: Obtaining public key for shared inbox')
+            personUrl = \
+                personUrl.replace(possibleUsersPath + 'inbox', '/inbox')
+            break
+    personJson = \
+        getPersonFromCache(baseDir, personUrl, personCache, True)
+    if not personJson:
+        if debug:
+            print('DEBUG: Obtaining public key for ' + personUrl)
+        personDomain = domain
+        if onionDomain:
+            if '.onion/' in personUrl:
+                personDomain = onionDomain
+        profileStr = 'https://www.w3.org/ns/activitystreams'
+        asHeader = {
+            'Accept': 'application/activity+json; profile="' + profileStr + '"'
+        }
+        personJson = \
+            getJson(session, personUrl, asHeader, None, debug,
+                    projectVersion, httpPrefix, personDomain)
+        if not personJson:
+            return None
+    pubKey = None
+    if personJson.get('publicKey'):
+        if personJson['publicKey'].get('publicKeyPem'):
+            pubKey = personJson['publicKey']['publicKeyPem']
+    else:
+        if personJson.get('publicKeyPem'):
+            pubKey = personJson['publicKeyPem']
+
+    if not pubKey:
+        if debug:
+            print('DEBUG: Public key not found for ' + personUrl)
+
+    storePersonInCache(baseDir, personUrl, personJson, personCache, True)
+    return pubKey

daemon.py

@@ -92,7 +92,6 @@ from inbox import runInboxQueue
 from inbox import runInboxQueueWatchdog
 from inbox import savePostToInboxQueue
 from inbox import populateReplies
-from inbox import getPersonPubKey
 from follow import isFollowingActor
 from follow import getFollowingFeed
 from follow import sendFollowRequest
@@ -273,7 +272,7 @@ from utils import isSuspended
 from utils import dangerousMarkup
 from utils import refreshNewswire
 from utils import isImageFile
-from utils import hasGroupPath
+from utils import hasGroupType
 from manualapprove import manualDenyFollowRequest
 from manualapprove import manualApproveFollowRequest
 from announce import createAnnounce
@@ -286,6 +285,7 @@ from media import processMetaData
 from cache import checkForChangedActor
 from cache import storePersonInCache
 from cache import getPersonFromCache
+from cache import getPersonPubKey
 from httpsig import verifyPostHeaders
 from theme import importTheme
 from theme import exportTheme
@@ -2569,7 +2569,9 @@ class PubServer(BaseHTTPRequestHandler):
                 }
                 pathUsersSection = path.split('/users/')[1]
                 self.postToNickname = pathUsersSection.split('/')[0]
-                groupAccount = hasGroupPath(followingActor)
+                groupAccount = hasGroupType(self.server.baseDir,
+                                            followingActor,
+                                            self.server.personCache)
                 unfollowAccount(self.server.baseDir, self.postToNickname,
                                 self.server.domain,
                                 followingNickname, followingDomainFull,

follow.py

@@ -28,13 +28,14 @@ from utils import saveJson
 from utils import isAccountDir
 from utils import getUserPaths
 from utils import acctDir
-from utils import hasGroupPath
+from utils import hasGroupType
 from acceptreject import createAccept
 from acceptreject import createReject
 from webfinger import webfingerHandle
 from auth import createBasicAuthHeader
 from session import getJson
 from session import postJson
+from cache import getPersonPubKey
 
 
 def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None:
@@ -626,7 +627,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str,
                          cachedWebfingers: {}, personCache: {},
                          messageJson: {}, federationList: [],
                          debug: bool, projectVersion: str,
-                         maxFollowers: int) -> bool:
+                         maxFollowers: int, onionDomain: str) -> bool:
     """Receives a follow request within the POST section of HTTPServer
     """
     if not messageJson['type'].startswith('Follow'):
@@ -740,22 +741,34 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str,
     else:
         print('Follow request does not require approval')
         # update the followers
-        if os.path.isdir(baseDir + '/accounts/' +
+        accountToBeFollowed = \
-                         nicknameToFollow + '@' + domainToFollow):
+            acctDir(baseDir, nicknameToFollow, domainToFollow)
-            followersFilename = \
+        if os.path.isdir(accountToBeFollowed):
-                baseDir + '/accounts/' + \
+            followersFilename = accountToBeFollowed + '/followers.txt'
-                nicknameToFollow + '@' + domainToFollow + '/followers.txt'
 
             # for actors which don't follow the mastodon
             # /users/ path convention store the full actor
             if '/users/' not in messageJson['actor']:
                 approveHandle = messageJson['actor']
 
+            # Get the actor for the follower and add it to the cache.
+            # Getting their public key has the same result
+            if debug:
+                print('Obtaining the following actor: ' + messageJson['actor'])
+            if not getPersonPubKey(baseDir, session, messageJson['actor'],
+                                   personCache, debug, projectVersion,
+                                   httpPrefix, domainToFollow, onionDomain):
+                if debug:
+                    print('Unable to obtain following actor: ' +
+                          messageJson['actor'])
+
             print('Updating followers file: ' +
                   followersFilename + ' adding ' + approveHandle)
             if os.path.isfile(followersFilename):
                 if approveHandle not in open(followersFilename).read():
-                    groupAccount = hasGroupPath(messageJson['object'])
+                    groupAccount = \
+                        hasGroupType(baseDir, messageJson['object'],
+                                     personCache)
                     try:
                         with open(followersFilename, 'r+') as followersFile:
                             content = followersFile.read()
@@ -925,7 +938,7 @@ def sendFollowRequest(session, baseDir: str,
     if followNickname:
         followedId = followedActor
         followHandle = followNickname + '@' + requestDomain
-        groupAccount = hasGroupPath(followedActor)
+        groupAccount = hasGroupType(baseDir, followedActor, personCache)
         if groupAccount:
             followHandle = '!' + followHandle
     else:
@@ -1435,7 +1448,7 @@ def outboxUndoFollow(baseDir: str, messageJson: {}, debug: bool) -> None:
         getDomainFromActor(messageJson['object']['object'])
     domainFollowingFull = getFullDomain(domainFollowing, portFollowing)
 
-    groupAccount = hasGroupPath(messageJson['object']['object'])
+    groupAccount = hasGroupType(baseDir, messageJson['object']['object'], None)
     if unfollowAccount(baseDir, nicknameFollower, domainFollowerFull,
                        nicknameFollowing, domainFollowingFull,
                        debug, groupAccount):

inbox.py

@@ -45,19 +45,18 @@ from utils import loadJson
 from utils import saveJson
 from utils import updateLikesCollection
 from utils import undoLikesCollectionEntry
-from utils import hasGroupPath
+from utils import hasGroupType
 from categories import getHashtagCategories
 from categories import setHashtagCategory
 from httpsig import verifyPostHeaders
 from session import createSession
-from session import getJson
 from follow import isFollowingActor
 from follow import receiveFollowRequest
 from follow import getFollowersOfActor
 from follow import unfollowerOfAccount
 from pprint import pprint
-from cache import getPersonFromCache
 from cache import storePersonInCache
+from cache import getPersonPubKey
 from acceptreject import receiveAcceptReject
 from bookmarks import updateBookmarksCollection
 from bookmarks import undoBookmarksCollectionEntry
@@ -234,55 +233,6 @@ def validInboxFilenames(baseDir: str, nickname: str, domain: str,
     return True
 
 
-def getPersonPubKey(baseDir: str, session, personUrl: str,
-                    personCache: {}, debug: bool,
-                    projectVersion: str, httpPrefix: str,
-                    domain: str, onionDomain: str) -> str:
-    if not personUrl:
-        return None
-    personUrl = personUrl.replace('#main-key', '')
-    usersPaths = getUserPaths()
-    for possibleUsersPath in usersPaths:
-        if personUrl.endswith(possibleUsersPath + 'inbox'):
-            if debug:
-                print('DEBUG: Obtaining public key for shared inbox')
-            personUrl = \
-                personUrl.replace(possibleUsersPath + 'inbox', '/inbox')
-            break
-    personJson = \
-        getPersonFromCache(baseDir, personUrl, personCache, True)
-    if not personJson:
-        if debug:
-            print('DEBUG: Obtaining public key for ' + personUrl)
-        personDomain = domain
-        if onionDomain:
-            if '.onion/' in personUrl:
-                personDomain = onionDomain
-        profileStr = 'https://www.w3.org/ns/activitystreams'
-        asHeader = {
-            'Accept': 'application/activity+json; profile="' + profileStr + '"'
-        }
-        personJson = \
-            getJson(session, personUrl, asHeader, None, debug,
-                    projectVersion, httpPrefix, personDomain)
-        if not personJson:
-            return None
-    pubKey = None
-    if personJson.get('publicKey'):
-        if personJson['publicKey'].get('publicKeyPem'):
-            pubKey = personJson['publicKey']['publicKeyPem']
-    else:
-        if personJson.get('publicKeyPem'):
-            pubKey = personJson['publicKeyPem']
-
-    if not pubKey:
-        if debug:
-            print('DEBUG: Public key not found for ' + personUrl)
-
-    storePersonInCache(baseDir, personUrl, personJson, personCache, True)
-    return pubKey
-
-
 def inboxMessageHasParams(messageJson: {}) -> bool:
     """Checks whether an incoming message contains expected parameters
     """
@@ -676,7 +626,7 @@ def _receiveUndoFollow(session, baseDir: str, httpPrefix: str,
         getDomainFromActor(messageJson['object']['object'])
     domainFollowingFull = getFullDomain(domainFollowing, portFollowing)
 
-    groupAccount = hasGroupPath(messageJson['object']['actor'])
+    groupAccount = hasGroupType(baseDir, messageJson['object']['actor'], None)
     if unfollowerOfAccount(baseDir,
                            nicknameFollowing, domainFollowingFull,
                            nicknameFollower, domainFollowerFull,
@@ -3148,7 +3098,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
                                 queueJson['post'],
                                 federationList,
                                 debug, projectVersion,
-                                maxFollowers):
+                                maxFollowers, onionDomain):
             if os.path.isfile(queueFilename):
                 os.remove(queueFilename)
             if len(queue) > 0:

migrate.py

@@ -12,7 +12,7 @@ from utils import isAccountDir
 from utils import getNicknameFromActor
 from utils import getDomainFromActor
 from utils import acctDir
-from utils import hasGroupPath
+from utils import hasGroupType
 from webfinger import webfingerHandle
 from blocking import isBlocked
 from posts import getUserUrl
@@ -103,7 +103,7 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str,
     if movedToPort:
         if movedToPort != 80 and movedToPort != 443:
             movedToDomainFull = movedToDomain + ':' + str(movedToPort)
-    groupAccount = hasGroupPath(movedToUrl)
+    groupAccount = hasGroupType(baseDir, movedToUrl, None)
     if isBlocked(baseDir, nickname, domain,
                  movedToNickname, movedToDomain):
         # someone that you follow has moved to a blocked domain

tests.py

@@ -39,6 +39,7 @@ from follow import clearFollowers
 from follow import sendFollowRequestViaServer
 from follow import sendUnfollowRequestViaServer
 from siteactive import siteIsActive
+from utils import isGroupActor
 from utils import dateStringToSeconds
 from utils import dateSecondsToString
 from utils import validPassword
@@ -1261,6 +1262,7 @@ def testFollowBetweenServers():
     assert 'bob@' + bobDomain in open(aliceDir + '/accounts/alice@' +
                                       aliceDomain +
                                       '/followingCalendar.txt').read()
+    assert not isGroupActor(aliceDir, bobActor, alicePersonCache)
 
     print('\n\n*********************************************************')
     print('Alice sends a message to Bob')
@@ -1319,7 +1321,10 @@ def testGroupFollow():
     print('Testing following of a group')
 
     global testServerAliceRunning
+    global testServerGroupRunning
+    systemLanguage = 'en'
     testServerAliceRunning = False
+    testServerGroupRunning = False
 
     # systemLanguage = 'en'
     httpPrefix = 'http'
@@ -1399,17 +1404,18 @@ def testGroupFollow():
     print('Alice sends a follow request to the test group')
     os.chdir(aliceDir)
     sessionAlice = createSession(proxyType)
-    # inReplyTo = None
+    inReplyTo = None
-    # inReplyToAtomUri = None
+    inReplyToAtomUri = None
-    # subject = None
+    subject = None
     alicePostLog = []
-    # followersOnly = False
+    followersOnly = False
-    # saveToFile = True
+    saveToFile = True
     clientToServer = False
-    # ccUrl = None
+    ccUrl = None
     alicePersonCache = {}
     aliceCachedWebfingers = {}
     alicePostLog = []
+    # aliceActor = httpPrefix + '://' + aliceAddress + '/users/alice'
     testgroupActor = httpPrefix + '://' + testgroupAddress + '/users/testgroup'
     sendResult = \
         sendFollowRequest(sessionAlice, aliceDir,
@@ -1422,32 +1428,88 @@ def testGroupFollow():
                           True, __version__)
     print('sendResult: ' + str(sendResult))
 
+    aliceFollowingFilename = \
+        aliceDir + '/accounts/alice@' + aliceDomain + '/following.txt'
+    aliceFollowingCalendarFilename = \
+        aliceDir + '/accounts/alice@' + aliceDomain + \
+        '/followingCalendar.txt'
+    testgroupFollowersFilename = \
+        testgroupDir + '/accounts/testgroup@' + testgroupDomain + \
+        '/followers.txt'
+
     for t in range(16):
-        if os.path.isfile(testgroupDir + '/accounts/testgroup@' +
+        if os.path.isfile(testgroupFollowersFilename):
-                          testgroupDomain + '/followers.txt'):
+            if os.path.isfile(aliceFollowingFilename):
-            if os.path.isfile(aliceDir + '/accounts/alice@' +
+                if os.path.isfile(aliceFollowingCalendarFilename):
-                              aliceDomain + '/following.txt'):
-                if os.path.isfile(aliceDir + '/accounts/alice@' +
-                                  aliceDomain + '/followingCalendar.txt'):
                     break
         time.sleep(1)
 
     assert validInbox(testgroupDir, 'testgroup', testgroupDomain)
     assert validInboxFilenames(testgroupDir, 'testgroup', testgroupDomain,
                                aliceDomain, alicePort)
-    assert 'alice@' + aliceDomain in open(testgroupDir +
+    assert 'alice@' + aliceDomain in open(testgroupFollowersFilename).read()
-                                          '/accounts/testgroup@' +
+
-                                          testgroupDomain +
+    testgroupWebfingerFilename = \
-                                          '/followers.txt').read()
+        testgroupDir + '/wfendpoints/testgroup@' + \
-    assert 'testgroup@' + testgroupDomain in open(aliceDir +
+        testgroupDomain + ':' + str(testgroupPort) + '.json'
-                                                  '/accounts/alice@' +
+    assert os.path.isfile(testgroupWebfingerFilename)
-                                                  aliceDomain +
+    assert 'group:testgroup@' in open(testgroupWebfingerFilename).read()
-                                                  '/following.txt').read()
+    print('group: exists within the webfinger endpoint for testgroup')
+
     testgroupHandle = 'testgroup@' + testgroupDomain
-    assert testgroupHandle in open(aliceDir +
+    followingStr = ''
-                                   '/accounts/alice@' +
+    with open(aliceFollowingFilename, 'r') as fp:
-                                   aliceDomain +
+        followingStr = fp.read()
-                                   '/followingCalendar.txt').read()
+        print('Alice following.txt:\n\n' + followingStr)
+    if '!testgroup' not in followingStr:
+        print('Alice following.txt does not contain !testgroup@' +
+              testgroupDomain + ':' + str(testgroupPort))
+    assert isGroupActor(aliceDir, testgroupActor, alicePersonCache)
+    assert '!testgroup' in followingStr
+    assert testgroupHandle in open(aliceFollowingFilename).read()
+    assert testgroupHandle in open(aliceFollowingCalendarFilename).read()
+    print('Alice follows the test group')
+
+    print('\n\n*********************************************************')
+    print('Alice posts to the test group')
+    alicePostLog = []
+    alicePersonCache = {}
+    aliceCachedWebfingers = {}
+    alicePostLog = []
+    isArticle = False
+    city = 'London, England'
+    sendResult = \
+        sendPost(__version__,
+                 sessionAlice, aliceDir, 'alice', aliceDomain, alicePort,
+                 'testgroup', testgroupDomain, testgroupPort, ccUrl,
+                 httpPrefix, "Alice group message", followersOnly,
+                 saveToFile, clientToServer, True,
+                 None, None, None, city, federationList,
+                 aliceSendThreads, alicePostLog, aliceCachedWebfingers,
+                 alicePersonCache, isArticle, systemLanguage, inReplyTo,
+                 inReplyToAtomUri, subject)
+    print('sendResult: ' + str(sendResult))
+
+    queuePath = \
+        testgroupDir + '/accounts/testgroup@' + testgroupDomain + '/queue'
+    inboxPath = \
+        testgroupDir + '/accounts/testgroup@' + testgroupDomain + '/inbox'
+    aliceMessageArrived = False
+    startPosts = len([name for name in os.listdir(inboxPath)
+                      if os.path.isfile(os.path.join(inboxPath, name))])
+    for i in range(20):
+        time.sleep(1)
+        if os.path.isdir(inboxPath):
+            currPosts = \
+                len([name for name in os.listdir(inboxPath)
+                     if os.path.isfile(os.path.join(inboxPath, name))])
+            if currPosts > startPosts:
+                aliceMessageArrived = True
+                print('Alice post sent to test group!')
+                break
+
+    assert aliceMessageArrived is True
+    print('Post from Alice to test group succeeded')
 
     # stop the servers
     thrAlice.kill()
@@ -1465,6 +1527,7 @@ def testGroupFollow():
 
     os.chdir(baseDir)
     shutil.rmtree(baseDir + '/.tests')
+    print('Testing following of a group is complete')
 
 
 def _testFollowersOfPerson():

utils.py

@@ -1081,6 +1081,9 @@ def followPerson(baseDir: str, nickname: str, domain: str,
     else:
         handleToFollow = followNickname + '@' + followDomain
 
+    if groupAccount:
+        handleToFollow = '!' + handleToFollow
+
     # was this person previously unfollowed?
     unfollowedFilename = baseDir + '/accounts/' + handle + '/unfollowed.txt'
     if os.path.isfile(unfollowedFilename):
@@ -1098,6 +1101,8 @@ def followPerson(baseDir: str, nickname: str, domain: str,
     if not os.path.isdir(baseDir + '/accounts'):
         os.mkdir(baseDir + '/accounts')
     handleToFollow = followNickname + '@' + followDomain
+    if groupAccount:
+        handleToFollow = '!' + handleToFollow
     filename = baseDir + '/accounts/' + handle + '/' + followFile
     if os.path.isfile(filename):
         if handleToFollow in open(filename).read():
@@ -2682,13 +2687,33 @@ def dateSecondsToString(dateSec: int) -> str:
     return thisDate.strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
-def hasGroupPath(actor: str) -> bool:
+def hasGroupType(baseDir: str, actor: str, personCache: {}) -> bool:
-    """Does the given actor url contain a path which indicates
+    """Does the given actor url have a group type?
-    that it belongs to a group?
-    e.g. https://lemmy/c/groupname
     """
+    # does the actor path clearly indicate that this is a group?
+    # eg. https://lemmy/c/groupname
     groupPaths = getGroupPaths()
     for grpPath in groupPaths:
         if grpPath in actor:
             return True
+    # is there a cached actor which can be examined for Group type?
+    return isGroupActor(baseDir, actor, personCache)
+
+
+def isGroupActor(baseDir: str, actor: str, personCache: {}) -> bool:
+    """Is the given actor a group?
+    """
+    if personCache:
+        if personCache.get(actor):
+            if personCache[actor].get('actor'):
+                if personCache[actor]['actor'].get('type'):
+                    if personCache[actor]['actor']['type'] == 'Group':
+                        return True
+                return False
+    cachedActorFilename = \
+        baseDir + '/cache/actors/' + (actor.replace('/', '#')) + '.json'
+    if not os.path.isfile(cachedActorFilename):
+        return False
+    if '"type": "Group"' in open(cachedActorFilename).read():
+        return True
     return False