Add migrations option to check for moved follows

epicyon.py

@@ -73,6 +73,7 @@ from shares import addShare
 from theme import setTheme
 from announce import sendAnnounceViaServer
 from socnet import instancesGraph
+from migrate import migrateAccounts
 import argparse
 
 
@@ -321,6 +322,9 @@ parser.add_argument("--i2p", type=str2bool, nargs='?',
 parser.add_argument("--tor", type=str2bool, nargs='?',
                     const=True, default=False,
                     help="Route via Tor")
+parser.add_argument("--migrations", type=str2bool, nargs='?',
+                    const=True, default=False,
+                    help="Migrate moved accounts")
 parser.add_argument("--tests", type=str2bool, nargs='?',
                     const=True, default=False,
                     help="Run unit tests")
@@ -1304,6 +1308,33 @@ if args.hyper:
 if args.i2p:
     httpPrefix = 'http'
 
+if args.migrations:
+    cachedWebfingers = {}
+    if args.http or domain.endswith('.onion'):
+        httpPrefix = 'http'
+        port = 80
+        proxyType = 'tor'
+    elif domain.endswith('.i2p'):
+        httpPrefix = 'http'
+        port = 80
+        proxyType = 'i2p'
+    elif args.gnunet:
+        httpPrefix = 'gnunet'
+        port = 80
+        proxyType = 'gnunet'
+    else:
+        httpPrefix = 'https'
+        port = 443
+    session = createSession(proxyType)
+    ctr = migrateAccounts(baseDir, session,
+                          httpPrefix, cachedWebfingers,
+                          True)
+    if ctr == 0:
+        print('No followed accounts have moved')
+    else:
+        print(str(ctr) + ' followed accounts were migrated')
+    sys.exit()
+
 if args.actor:
     originalActor = args.actor
     if '/@' in args.actor or \

migrate.py

@@ -7,48 +7,175 @@ __email__ = "bob@freedombone.net"
 __status__ = "Production"
 
 import os
+from utils import getNicknameFromActor
+from utils import getDomainFromActor
+from webfinger import webfingerHandle
+from blocking import isBlocked
+from session import getJson
+from posts import getUserUrl
+from follow import unfollowAccount
 
 
-def _migrateFollows(followFilename: str, oldHandle: str,
+def _moveFollowingHandlesForAccount(baseDir: str, nickname: str, domain: str,
-                    newHandle: str) -> None:
+                                    session,
-    """Changes a handle within following or followers list
+                                    httpPrefix: str, cachedWebfingers: {},
+                                    followFile: str, debug: bool) -> int:
+    """Goes through all follows for an account and updates any that have moved
     """
-    if not os.path.isfile(followFilename):
+    ctr = 0
-        return
+    followingFilename = \
-    if oldHandle not in open(followFilename).read():
+        baseDir + '/accounts/' + nickname + '@' + domain + '/' + followFile
-        return
+    if not os.path.isfile(followingFilename):
-    followData = None
+        return ctr
-    with open(followFilename, 'r') as followFile:
+    with open(followingFilename, "r") as f:
-        followData = followFile.read()
+        followingHandles = f.readlines()
-    if not followData:
+        for followHandle in followingHandles:
-        return
+            followHandle = followHandle.strip("\n").strip("\r")
-    newFollowData = followData.replace(oldHandle, newHandle)
+            ctr += \
-    if followData == newFollowData:
+                _updateMovedHandle(baseDir, nickname, domain,
-        return
+                                   followHandle, session,
-    with open(followFilename, 'w+') as followFile:
+                                   httpPrefix, cachedWebfingers,
-        followFile.write(newFollowData)
+                                   followFile, debug)
+    return ctr
 
 
-def migrateAccount(baseDir: str, oldHandle: str, newHandle: str) -> None:
+def _updateMovedHandle(baseDir: str, nickname: str, domain: str,
-    """If a followed account changes then this modifies the
+                       handle: str, session,
-    following and followers lists for each account accordingly
+                       httpPrefix: str, cachedWebfingers: {},
+                       followFile: str, debug: bool) -> int:
+    """Check if an account has moved, and if so then alter following.txt
+    for each account.
+    Returns 1 if moved, 0 otherwise
     """
-    if oldHandle.startswith('@'):
+    ctr = 0
-        oldHandle = oldHandle[1:]
+    if '@' not in handle:
-    if '@' not in oldHandle:
+        return ctr
-        return
+    if len(handle) < 5:
-    if newHandle.startswith('@'):
+        return ctr
-        newHandle = newHandle[1:]
+    if handle.startswith('@'):
-    if '@' not in newHandle:
+        handle = handle[1:]
-        return
+    wfRequest = webfingerHandle(session, handle,
+                                httpPrefix, cachedWebfingers,
+                                None, __version__)
+    if not wfRequest:
+        print('updateMovedHandle unable to webfinger ' + handle)
+        return ctr
 
+    if not isinstance(wfRequest, dict):
+        print('updateMovedHandle webfinger for ' + handle +
+              ' did not return a dict. ' + str(wfRequest))
+        return ctr
+
+    personUrl = None
+    if wfRequest.get('errors'):
+        print('wfRequest error: ' + str(wfRequest['errors']))
+        return ctr
+
+    profileStr = 'https://www.w3.org/ns/activitystreams'
+    asHeader = {
+        'Accept': 'application/activity+json; profile="' + profileStr + '"'
+    }
+    if not personUrl:
+        personUrl = getUserUrl(wfRequest)
+        if not personUrl:
+            return ctr
+
+    profileStr = 'https://www.w3.org/ns/activitystreams'
+    asHeader = {
+        'Accept': 'application/ld+json; profile="' + profileStr + '"'
+    }
+    personJson = \
+        getJson(session, personUrl, asHeader, None, __version__,
+                httpPrefix, None)
+    if not personJson:
+        return ctr
+    if not personJson.get('movedTo'):
+        return ctr
+    movedToUrl = personJson['movedTo']
+    if '://' not in movedToUrl:
+        return ctr
+    if '.' not in movedToUrl:
+        return ctr
+    movedToNickname = getNicknameFromActor(movedToUrl)
+    if not movedToNickname:
+        return ctr
+    movedToDomain, movedToPort = getDomainFromActor(movedToUrl)
+    if not movedToDomain:
+        return ctr
+    movedToDomainFull = movedToDomain
+    if movedToPort:
+        if movedToPort != 80 and movedToPort != 443:
+            movedToDomainFull = movedToDomain + ':' + str(movedToPort)
+    if isBlocked(baseDir, nickname, domain,
+                 movedToNickname, movedToDomain):
+        # someone that you follow has moved to a blocked domain
+        # so just unfollow them
+        unfollowAccount(baseDir, nickname, domain,
+                        movedToNickname, movedToDomainFull,
+                        followFile, debug)
+        return ctr
+    followingFilename = \
+        baseDir + '/accounts/' + nickname + '@' + domain + '/' + followFile
+    if not os.path.isfile(followingFilename):
+        return ctr
+    with open(followingFilename, "r") as f:
+        followingHandles = f.readlines()
+        movedToHandle = movedToNickname + '@' + movedToDomainFull
+        movedToHandleLower = movedToHandle.lower()
+        handleLower = handle.lower()
+        # does the new handle already exist in the following list?
+        alreadyFollowingHandle = False
+        for followHandle in followingHandles:
+            if followHandle.strip("\n").strip("\r").lower() == \
+               movedToHandleLower:
+                alreadyFollowingHandle = True
+        if not alreadyFollowingHandle:
+            # replace the old handle with the new one
+            with open(followingFilename, 'w+') as f:
+                for followHandle in followingHandles:
+                    if followHandle.strip("\n").strip("\r").lower() != \
+                       handleLower:
+                        f.write(followHandle)
+                    else:
+                        f.write(movedToHandleLower + '\n')
+                        ctr += 1
+                        print('Follow moved from ' + handleLower +
+                              ' to ' + movedToHandleLower)
+        else:
+            # remove the old handle
+            with open(followingFilename, 'w+') as f:
+                for followHandle in followingHandles:
+                    if followHandle.strip("\n").strip("\r").lower() != \
+                       handleLower:
+                        f.write(followHandle)
+                    else:
+                        ctr += 1
+    return ctr
+
+
+def migrateAccounts(baseDir: str, session,
+                    httpPrefix: str, cachedWebfingers: {},
+                    debug: bool) -> int:
+    """If followed accounts change then this modifies the
+    following lists for each account accordingly.
+    Returns the number of accounts migrated
+    """
     # update followers and following lists for each account
+    ctr = 0
     for subdir, dirs, files in os.walk(baseDir + '/accounts'):
         for handle in dirs:
-            if '@' in handle:
+            if '@' not in handle:
-                accountDir = baseDir + '/accounts/' + handle
+                continue
-                followFilename = accountDir + '/following.txt'
+            if handle.startswith('inbox@'):
-                _migrateFollows(followFilename, oldHandle, newHandle)
+                continue
-                followFilename = accountDir + '/followers.txt'
+            if handle.startswith('news@'):
-                _migrateFollows(followFilename, oldHandle, newHandle)
+                continue
+            nickname = handle.split('@')[0]
+            domain = handle.split('@')[1]
+            ctr += \
+                _moveFollowingHandlesForAccount(baseDir, nickname, domain,
+                                                session, httpPrefix,
+                                                cachedWebfingers,
+                                                'following.txt', debug)
         break
+    return ctr

tests.py

@@ -2799,7 +2799,6 @@ def testFunctions():
         'threadSendPost',
         'sendToFollowers',
         'expireCache',
-        'migrateAccount',
         'getMutualsOfPerson',
         'runPostsQueue',
         'runSharesExpire',