From 8c2b1e3018a9cd4bab0d1ec5cb1ebb1bd0e76f39 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 11:53:13 +0000 Subject: [PATCH 01/22] Captalize header parts --- httpsig.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/httpsig.py b/httpsig.py index bb2997f03..192658648 100644 --- a/httpsig.py +++ b/httpsig.py @@ -353,7 +353,15 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, signedHeaderList.append( f'{signedHeader}: {headers[signedHeader]}') else: - signedHeaderCap = signedHeader.capitalize() + if '-' in signedHeader: + headerParts = signedHeader.split('-') + signedHeaderCap = '' + for part in headerParts: + if signedHeaderCap: + signedHeaderCap += '-' + signedHeaderCap += part.capitalize() + else: + signedHeaderCap = signedHeader.capitalize() if signedHeaderCap == 'Date': if not _verifyRecentSignature(headers[signedHeaderCap]): if debug: From 9ddda0b9e04ce752db37ee31aa255de6919ea35a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 12:09:56 +0000 Subject: [PATCH 02/22] More debug --- httpsig.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/httpsig.py b/httpsig.py index 192658648..077e95a10 100644 --- a/httpsig.py +++ b/httpsig.py @@ -354,14 +354,23 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, f'{signedHeader}: {headers[signedHeader]}') else: if '-' in signedHeader: + # capitalise with dashes + # my-header becomes My-Header headerParts = signedHeader.split('-') - signedHeaderCap = '' + signedHeaderCap = None for part in headerParts: if signedHeaderCap: - signedHeaderCap += '-' - signedHeaderCap += part.capitalize() + signedHeaderCap += '-' + part.capitalize() + else: + signedHeaderCap = part.capitalize() else: + # header becomes Header signedHeaderCap = signedHeader.capitalize() + + if debug: + print('signedHeaderCap: ' + signedHeaderCap) + + # if this is the date header then check it is recent if signedHeaderCap == 'Date': if not _verifyRecentSignature(headers[signedHeaderCap]): if debug: @@ -369,6 +378,8 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, 'verifyPostHeaders date is not recent ' + headers[signedHeader]) return False + + # add the capitalised header if headers.get(signedHeaderCap): signedHeaderList.append( f'{signedHeader}: {headers[signedHeaderCap]}') From 8bf9607cda08f9bbddd7b947f01c0aba1a668557 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 12:18:05 +0000 Subject: [PATCH 03/22] Also log headers --- httpsig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httpsig.py b/httpsig.py index 077e95a10..c89227e97 100644 --- a/httpsig.py +++ b/httpsig.py @@ -368,7 +368,8 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, signedHeaderCap = signedHeader.capitalize() if debug: - print('signedHeaderCap: ' + signedHeaderCap) + print('signedHeaderCap: ' + + signedHeaderCap + ' ' + str(headers)) # if this is the date header then check it is recent if signedHeaderCap == 'Date': From 77b6d76c42fb9bb1bdeb0d820684681ccdea8f6c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 12:24:01 +0000 Subject: [PATCH 04/22] Another permitted signature header --- daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon.py b/daemon.py index 7f16a4f7b..c483efc69 100644 --- a/daemon.py +++ b/daemon.py @@ -1248,6 +1248,9 @@ class PubServer(BaseHTTPRequestHandler): headersDict['Date'] = self.headers['Date'] if self.headers.get('digest'): headersDict['digest'] = self.headers['digest'] + if self.headers.get('Collection-Synchronization'): + headersDict['Collection-Synchronization'] = \ + self.headers['Collection-Synchronization'] if self.headers.get('Content-type'): headersDict['Content-type'] = self.headers['Content-type'] if self.headers.get('Content-Length'): From 94c4b31e3996961a7e9bbb709687e24152c2fe90 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 15:00:43 +0000 Subject: [PATCH 05/22] Extra debug on http verify --- httpsig.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/httpsig.py b/httpsig.py index c89227e97..24c94f5bf 100644 --- a/httpsig.py +++ b/httpsig.py @@ -265,6 +265,8 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, if debug: print('DEBUG: verifyPostHeaders ' + method) + print('verifyPostHeaders publicKeyPem: ' + str(publicKeyPem)) + print('verifyPostHeaders headers: ' + str(headers)) pubkey = load_pem_public_key(publicKeyPem.encode('utf-8'), backend=default_backend()) @@ -368,8 +370,7 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, signedHeaderCap = signedHeader.capitalize() if debug: - print('signedHeaderCap: ' + - signedHeaderCap + ' ' + str(headers)) + print('signedHeaderCap: ' + signedHeaderCap) # if this is the date header then check it is recent if signedHeaderCap == 'Date': From 7345381f85374de17c0157ea3272e085cb4aff80 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 15:25:49 +0000 Subject: [PATCH 06/22] Debug --- httpsig.py | 2 ++ tests.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/httpsig.py b/httpsig.py index 24c94f5bf..c90074e83 100644 --- a/httpsig.py +++ b/httpsig.py @@ -267,6 +267,8 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, print('DEBUG: verifyPostHeaders ' + method) print('verifyPostHeaders publicKeyPem: ' + str(publicKeyPem)) print('verifyPostHeaders headers: ' + str(headers)) + print('verifyPostHeaders messageBodyJsonStr: ' + + str(messageBodyJsonStr)) pubkey = load_pem_public_key(publicKeyPem.encode('utf-8'), backend=default_backend()) diff --git a/tests.py b/tests.py index a3ba76b63..f204295ff 100644 --- a/tests.py +++ b/tests.py @@ -279,6 +279,13 @@ def testHttpSigNew(): 'cvhBfgn/xeAsNxz67SIHMgiXvLL6TFqEI1en9dl9A3ihB6ZO6+W' + \ 'gUoW7OobZNlPxAUkQCc2A6oVjCYOdpKdrMAXQp2TQQ==:' +# publicKeyPem = '' + + assert verifyPostHeaders(httpPrefix, publicKeyPem, headers, + boxpath, False, None, + messageBodyJsonStr, debug, + True) + def _testHttpsigBase(withDigest): print('testHttpsig(' + str(withDigest) + ')') From 840aca1808dc0a14e332d90ceaae840ebf702a3c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 17:53:47 +0000 Subject: [PATCH 07/22] Allow collection synchronization within http headers --- tests.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests.py b/tests.py index f204295ff..a3ba76b63 100644 --- a/tests.py +++ b/tests.py @@ -279,13 +279,6 @@ def testHttpSigNew(): 'cvhBfgn/xeAsNxz67SIHMgiXvLL6TFqEI1en9dl9A3ihB6ZO6+W' + \ 'gUoW7OobZNlPxAUkQCc2A6oVjCYOdpKdrMAXQp2TQQ==:' -# publicKeyPem = '' - - assert verifyPostHeaders(httpPrefix, publicKeyPem, headers, - boxpath, False, None, - messageBodyJsonStr, debug, - True) - def _testHttpsigBase(withDigest): print('testHttpsig(' + str(withDigest) + ')') From 116a696543a22678923d0d497dbe70c579c226b7 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 18:02:32 +0000 Subject: [PATCH 08/22] Don't wait for keypress if running notification client within systemd daemon --- epicyon.py | 6 ++++++ install-desktop-notifications | 2 +- notifications_client.py | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/epicyon.py b/epicyon.py index cde1c49e8..707ed1163 100644 --- a/epicyon.py +++ b/epicyon.py @@ -295,6 +295,11 @@ parser.add_argument("--brochMode", type=str2bool, nargs='?', const=True, default=False, help="Enable broch mode") +parser.add_argument("--noKeyPress", + dest='noKeyPress', + type=str2bool, nargs='?', + const=True, default=False, + help="Notification daemon does not wait for keypresses") parser.add_argument("--noapproval", type=str2bool, nargs='?', const=True, default=False, help="Allow followers without approval") @@ -1857,6 +1862,7 @@ if args.notifications: args.screenreader, args.language, args.notificationSounds, args.notificationType, + args.noKeyPress, args.debug) sys.exit() diff --git a/install-desktop-notifications b/install-desktop-notifications index b52b36bc0..d3853110b 100755 --- a/install-desktop-notifications +++ b/install-desktop-notifications @@ -78,7 +78,7 @@ fi echo ''; echo '[Service]'; echo "WorkingDirectory=${HOME}/.epicyon"; - echo "ExecStart=/usr/bin/python3 epicyon.py --notifyType $notificationType --notify $HANDLE --password \"$PASSWORD\""; + echo "ExecStart=/usr/bin/python3 epicyon.py --noKeyPress --notifyType $notificationType --notify $HANDLE --password \"$PASSWORD\""; echo 'Type=oneshot'; echo 'RemainAfterExit=yes'; echo ''; diff --git a/notifications_client.py b/notifications_client.py index a3934aa7d..ef1798767 100644 --- a/notifications_client.py +++ b/notifications_client.py @@ -574,6 +574,7 @@ def runNotificationsClient(baseDir: str, proxyType: str, httpPrefix: str, systemLanguage: str, notificationSounds: bool, notificationType: str, + noKeyPress: bool, debug: bool) -> None: """Runs the notifications and screen reader client, which announces new inbox items @@ -772,7 +773,10 @@ def runNotificationsClient(baseDir: str, proxyType: str, httpPrefix: str, prevSay = speakerJson['say'] # wait for a while, or until a key is pressed - keyPress = _waitForKeypress(30, debug) + if noKeyPress: + time.sleep(10) + else: + keyPress = _waitForKeypress(30, debug) if keyPress: if keyPress.startswith('/'): keyPress = keyPress[1:] From a19822ce5af48e0dc73a098372353b532b2ec8b6 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 18:14:18 +0000 Subject: [PATCH 09/22] Handle exception in jsonld signature check --- pyjsonld.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyjsonld.py b/pyjsonld.py index 36b44d0bb..6bdfd78c5 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -2378,8 +2378,13 @@ class JsonLdProcessor(object): # hash bnode paths path_namer = UniqueNamer('_:b') path_namer.get_name(bnode) - results.append(self._hash_paths( - bnode, bnodes, namer, path_namer)) + try: + bnode_path = self._hash_paths( + bnode, bnodes, namer, path_namer) + results.append(bnode_path) + except BaseException: + print('WARN: jsonld bnode_path failed') + pass # name bnodes in hash order cmp_hashes = cmp_to_key(lambda x, y: cmp(x['hash'], y['hash'])) From 5981ee16d896c1df401a41f3907050a588fbd705 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 18:16:24 +0000 Subject: [PATCH 10/22] Debug header on signature failure --- inbox.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inbox.py b/inbox.py index 2c1498fa4..1e92ed930 100644 --- a/inbox.py +++ b/inbox.py @@ -2888,8 +2888,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, debug): httpSignatureFailed = True print('Queue: Header signature check failed') - if debug: - pprint(queueJson['httpHeaders']) + pprint(queueJson['httpHeaders']) else: if debug: print('DEBUG: http header signature check success') From c7533880777e035cedf0db41ad7fe32d35bd831c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 18:29:10 +0000 Subject: [PATCH 11/22] Another capitalization variant --- httpsig.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/httpsig.py b/httpsig.py index c90074e83..842a452db 100644 --- a/httpsig.py +++ b/httpsig.py @@ -387,6 +387,11 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, if headers.get(signedHeaderCap): signedHeaderList.append( f'{signedHeader}: {headers[signedHeaderCap]}') + else: + signedHeaderCap = signedHeader.capitalize() + if headers.get(signedHeaderCap): + signedHeaderList.append( + f'{signedHeader}: {headers[signedHeaderCap]}') if debug: print('DEBUG: signedHeaderList: ' + str(signedHeaderList)) From df0179768f47be7fe57a0795823554eca9e1b87f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 18:34:30 +0000 Subject: [PATCH 12/22] More efficient --- httpsig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httpsig.py b/httpsig.py index 842a452db..fca42310c 100644 --- a/httpsig.py +++ b/httpsig.py @@ -387,7 +387,8 @@ def verifyPostHeaders(httpPrefix: str, publicKeyPem: str, headers: dict, if headers.get(signedHeaderCap): signedHeaderList.append( f'{signedHeader}: {headers[signedHeaderCap]}') - else: + elif '-' in signedHeader: + # my-header becomes My-header signedHeaderCap = signedHeader.capitalize() if headers.get(signedHeaderCap): signedHeaderList.append( From faadca51b69426f94581a59dae9a54926f9d19bb Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:22:58 +0000 Subject: [PATCH 13/22] Less verbose when not in debug --- announce.py | 2 +- availability.py | 2 +- daemon.py | 16 ++++++++++++---- delete.py | 2 +- epicyon.py | 2 +- follow.py | 4 ++-- inbox.py | 13 +++++++------ like.py | 4 ++-- migrate.py | 2 +- person.py | 5 +++-- posts.py | 27 ++++++++++++++------------- roles.py | 2 +- shares.py | 4 ++-- skills.py | 2 +- socnet.py | 2 +- webapp_post.py | 2 +- webapp_profile.py | 14 ++++++++------ webfinger.py | 20 ++++++++++++-------- 18 files changed, 71 insertions(+), 54 deletions(-) diff --git a/announce.py b/announce.py index e8a3f2133..60679e4de 100644 --- a/announce.py +++ b/announce.py @@ -219,7 +219,7 @@ def sendAnnounceViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/availability.py b/availability.py index 0b0f48847..5b88c4103 100644 --- a/availability.py +++ b/availability.py @@ -105,7 +105,7 @@ def sendAvailabilityViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/daemon.py b/daemon.py index c483efc69..d4f4f893a 100644 --- a/daemon.py +++ b/daemon.py @@ -1379,7 +1379,7 @@ class PubServer(BaseHTTPRequestHandler): if GETtimings.get(prevGetId): timeDiff = int(timeDiff - int(GETtimings[prevGetId])) GETtimings[currGetId] = str(timeDiff) - if logEvent: + if logEvent and self.server.debug: print('GET TIMING ' + currGetId + ' = ' + str(timeDiff)) def _benchmarkPOSTtimings(self, POSTstartTime, POSTtimings: [], @@ -1397,7 +1397,8 @@ class PubServer(BaseHTTPRequestHandler): if logEvent: ctr = 1 for timeDiff in POSTtimings: - print('POST TIMING|' + str(ctr) + '|' + timeDiff) + if self.server.debug: + print('POST TIMING|' + str(ctr) + '|' + timeDiff) ctr += 1 def _pathContainsBlogLink(self, baseDir: str, @@ -7351,6 +7352,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, actorJson['roles'], None, None) msg = msg.encode('utf-8') @@ -7439,6 +7441,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, actorJson['skills'], None, None) msg = msg.encode('utf-8') @@ -9243,6 +9246,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, shares, pageNumber, sharesPerPage) msg = msg.encode('utf-8') @@ -9341,6 +9345,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, following, pageNumber, followsPerPage).encode('utf-8') @@ -9439,6 +9444,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, followers, pageNumber, followsPerPage).encode('utf-8') @@ -9560,6 +9566,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.peertubeInstances, self.server.allowLocalNetworkAccess, self.server.textModeBanner, + self.server.debug, None, None).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -13637,7 +13644,7 @@ class PubServer(BaseHTTPRequestHandler): # check authorization authorized = self._isAuthorized() - if not authorized: + if not authorized and self.server.debug: print('POST Not authorized') print(str(self.headers)) @@ -14236,7 +14243,8 @@ class PubServer(BaseHTTPRequestHandler): return else: if self.path == '/sharedInbox' or self.path == '/inbox': - print('DEBUG: POST to shared inbox') + if self.server.debug: + print('DEBUG: POST to shared inbox') queueStatus = \ self._updateInboxQueue('inbox', messageJson, messageBytes) if queueStatus >= 0 and queueStatus <= 3: diff --git a/delete.py b/delete.py index 027221abb..737f574e8 100644 --- a/delete.py +++ b/delete.py @@ -55,7 +55,7 @@ def sendDeleteViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/epicyon.py b/epicyon.py index 707ed1163..f31ef781a 100644 --- a/epicyon.py +++ b/epicyon.py @@ -1477,7 +1477,7 @@ if args.followers: handle = nickname + '@' + domain wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - None, __version__) + None, __version__, debug) if not wfRequest: print('Unable to webfinger ' + handle) sys.exit() diff --git a/follow.py b/follow.py index e4d20d815..6404eda2a 100644 --- a/follow.py +++ b/follow.py @@ -989,7 +989,7 @@ def sendFollowRequestViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) @@ -1078,7 +1078,7 @@ def sendUnfollowRequestViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/inbox.py b/inbox.py index 1e92ed930..de3587042 100644 --- a/inbox.py +++ b/inbox.py @@ -2702,7 +2702,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queue.pop(0) continue - print('Loading queue item ' + queueFilename) + if debug: + print('Loading queue item ' + queueFilename) # Load the queue json queueJson = loadJson(queueFilename, 1) @@ -2829,7 +2830,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if accountMaxPostsPerDay > 0 or domainMaxPostsPerDay > 0: pprint(quotasDaily) - if queueJson.get('actor'): + if debug and queueJson.get('actor'): print('Obtaining public key for actor ' + queueJson['actor']) # Try a few times to obtain the public key @@ -3053,8 +3054,9 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, httpPrefix, domain, port, debug) if len(recipientsDict.items()) == 0 and \ len(recipientsDictFollowers.items()) == 0: - print('Queue: no recipients were resolved ' + - 'for post arriving in inbox') + if debug: + print('Queue: no recipients were resolved ' + + 'for post arriving in inbox') if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: @@ -3122,8 +3124,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, themeName) if debug: pprint(queueJson['post']) - - print('Queue: Queue post accepted') + print('Queue: Queue post accepted') if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: diff --git a/like.py b/like.py index 4dcc86ceb..acf243f52 100644 --- a/like.py +++ b/like.py @@ -167,7 +167,7 @@ def sendLikeViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) @@ -248,7 +248,7 @@ def sendUndoLikeViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/migrate.py b/migrate.py index 9d050f114..9c24dce82 100644 --- a/migrate.py +++ b/migrate.py @@ -56,7 +56,7 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, handle = handle[1:] wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - None, __version__) + None, __version__, debug) if not wfRequest: print('updateMovedHandle unable to webfinger ' + handle) return ctr diff --git a/person.py b/person.py index d08963e22..e4d3c3f01 100644 --- a/person.py +++ b/person.py @@ -1102,7 +1102,8 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, return True -def getActorJson(handle: str, http: bool, gnunet: bool, quiet=False) -> {}: +def getActorJson(handle: str, http: bool, gnunet: bool, + debug: bool, quiet=False) -> {}: """Returns the actor json """ originalActor = handle @@ -1175,7 +1176,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, quiet=False) -> {}: handle = nickname + '@' + domain wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - None, __version__) + None, __version__, debug) if not wfRequest: if not quiet: print('Unable to webfinger ' + handle) diff --git a/posts.py b/posts.py index 6636a9c17..676ea3e67 100644 --- a/posts.py +++ b/posts.py @@ -1935,7 +1935,7 @@ def sendPost(projectVersion: str, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: return 1 if not isinstance(wfRequest, dict): @@ -2052,7 +2052,7 @@ def sendPostViaServer(projectVersion: str, # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: webfinger failed for ' + handle) @@ -2249,7 +2249,7 @@ def sendSignedJson(postJsonObject: {}, session, baseDir: str, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: webfinger for ' + handle + ' failed') @@ -2538,7 +2538,8 @@ def sendToNamedAddresses(session, baseDir: str, personCache, debug, projectVersion) -def _hasSharedInbox(session, httpPrefix: str, domain: str) -> bool: +def _hasSharedInbox(session, httpPrefix: str, domain: str, + debug: bool) -> bool: """Returns true if the given domain has a shared inbox This tries the new and the old way of webfingering the shared inbox """ @@ -2547,9 +2548,8 @@ def _hasSharedInbox(session, httpPrefix: str, domain: str) -> bool: 'inbox@' + domain ] for handle in tryHandles: - wfRequest = webfingerHandle(session, handle, - httpPrefix, {}, - None, __version__) + wfRequest = webfingerHandle(session, handle, httpPrefix, {}, + None, __version__, debug) if wfRequest: if isinstance(wfRequest, dict): if not wfRequest.get('errors'): @@ -2634,7 +2634,8 @@ def sendToFollowers(session, baseDir: str, print('Sending post to followers domain is active: ' + followerDomainUrl) - withSharedInbox = _hasSharedInbox(session, httpPrefix, followerDomain) + withSharedInbox = _hasSharedInbox(session, httpPrefix, + followerDomain, debug) if debug: if withSharedInbox: print(followerDomain + ' has shared inbox') @@ -3463,7 +3464,7 @@ def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str, handle = httpPrefix + "://" + domainFull + "/@" + nickname wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: sys.exit() if not isinstance(wfRequest, dict): @@ -3505,7 +3506,7 @@ def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str, handle = httpPrefix + "://" + domainFull + "/@" + nickname wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: return domainList if not isinstance(wfRequest, dict): @@ -3585,7 +3586,7 @@ def getPublicPostInfo(session, baseDir: str, nickname: str, domain: str, handle = httpPrefix + "://" + domainFull + "/@" + nickname wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: return {} if not isinstance(wfRequest, dict): @@ -4148,7 +4149,7 @@ def sendBlockViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) @@ -4232,7 +4233,7 @@ def sendUndoBlockViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/roles.py b/roles.py index 7af860db5..218ff7d3a 100644 --- a/roles.py +++ b/roles.py @@ -310,7 +310,7 @@ def sendRoleViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - delegatorDomain, projectVersion) + delegatorDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/shares.py b/shares.py index cf726d1a6..239f79a70 100644 --- a/shares.py +++ b/shares.py @@ -358,7 +358,7 @@ def sendShareViaServer(baseDir, session, wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) @@ -457,7 +457,7 @@ def sendUndoShareViaServer(baseDir: str, session, # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - fromDomain, projectVersion) + fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/skills.py b/skills.py index dd8ac8eac..267c43f37 100644 --- a/skills.py +++ b/skills.py @@ -123,7 +123,7 @@ def sendSkillViaServer(baseDir: str, session, nickname: str, password: str, wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: announce webfinger failed for ' + handle) diff --git a/socnet.py b/socnet.py index 42fbfd072..30450c1ff 100644 --- a/socnet.py +++ b/socnet.py @@ -52,7 +52,7 @@ def instancesGraph(baseDir: str, handles: str, wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wfRequest: return dotGraphStr + '}\n' if not isinstance(wfRequest, dict): diff --git a/webapp_post.py b/webapp_post.py index 1cd2099b6..895908213 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1223,7 +1223,7 @@ def individualPostAsHtml(allowDownloads: bool, postActorWf = \ webfingerHandle(session, postActorHandle, httpPrefix, cachedWebfingers, - domain, __version__) + domain, __version__, False) avatarUrl2 = None displayName = None diff --git a/webapp_profile.py b/webapp_profile.py index 1a987244b..0a4902822 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -115,7 +115,7 @@ def htmlProfileAfterSearch(cssCache: {}, webfingerHandle(session, searchNickname + '@' + searchDomainFull, httpPrefix, cachedWebfingers, - domain, projectVersion) + domain, projectVersion, debug) if not wf: print('DEBUG: Unable to webfinger ' + searchNickname + '@' + searchDomainFull) @@ -481,6 +481,7 @@ def htmlProfile(rssIconAtTop: bool, peertubeInstances: [], allowLocalNetworkAccess: bool, textModeBanner: str, + debug: bool, extraJson=None, pageNumber=None, maxItemsPerPage=None) -> str: """Show the profile page as html @@ -816,7 +817,7 @@ def htmlProfile(rssIconAtTop: bool, cachedWebfingers, personCache, extraJson, projectVersion, ["unfollow"], selected, usersPath, pageNumber, maxItemsPerPage, - dormantMonths) + dormantMonths, debug) elif selected == 'followers': profileStr += \ _htmlProfileFollowing(translate, baseDir, httpPrefix, @@ -825,7 +826,7 @@ def htmlProfile(rssIconAtTop: bool, cachedWebfingers, personCache, extraJson, projectVersion, ["block"], selected, usersPath, pageNumber, - maxItemsPerPage, dormantMonths) + maxItemsPerPage, dormantMonths, debug) elif selected == 'roles': profileStr += \ _htmlProfileRoles(translate, nickname, domainFull, @@ -920,7 +921,7 @@ def _htmlProfileFollowing(translate: {}, baseDir: str, httpPrefix: str, feedName: str, actor: str, pageNumber: int, maxItemsPerPage: int, - dormantMonths: int) -> str: + dormantMonths: int, debug: bool) -> str: """Shows following on the profile screen """ profileStr = '' @@ -952,7 +953,7 @@ def _htmlProfileFollowing(translate: {}, baseDir: str, httpPrefix: str, domain, followingActor, authorized, nickname, httpPrefix, projectVersion, dormant, - buttons) + debug, buttons) if authorized and maxItemsPerPage and pageNumber: if len(followingJson['orderedItems']) >= maxItemsPerPage: @@ -1801,6 +1802,7 @@ def _individualFollowAsHtml(translate: {}, httpPrefix: str, projectVersion: str, dormant: bool, + debug: bool, buttons=[]) -> str: """An individual follow entry on the profile screen """ @@ -1817,7 +1819,7 @@ def _individualFollowAsHtml(translate: {}, followUrlWf = \ webfingerHandle(session, followUrlHandle, httpPrefix, cachedWebfingers, - domain, __version__) + domain, __version__, debug) (inboxUrl, pubKeyId, pubKey, fromPersonId, sharedInbox, diff --git a/webfinger.py b/webfinger.py index 29c63bad7..920f97af7 100644 --- a/webfinger.py +++ b/webfinger.py @@ -40,11 +40,13 @@ def _parseHandle(handle: str) -> (str, str): def webfingerHandle(session, handle: str, httpPrefix: str, cachedWebfingers: {}, - fromDomain: str, projectVersion: str) -> {}: + fromDomain: str, projectVersion: str, + debug: bool) -> {}: """Gets webfinger result for the given ActivityPub handle """ if not session: - print('WARN: No session specified for webfingerHandle') + if debug: + print('WARN: No session specified for webfingerHandle') return None nickname, domain = _parseHandle(handle) @@ -60,7 +62,8 @@ def webfingerHandle(session, handle: str, httpPrefix: str, wf = getWebfingerFromCache(nickname + '@' + wfDomain, cachedWebfingers) if wf: - print('Webfinger from cache: ' + str(wf)) + if debug: + print('Webfinger from cache: ' + str(wf)) return wf url = '{}://{}/.well-known/webfinger'.format(httpPrefix, domain) par = { @@ -81,11 +84,12 @@ def webfingerHandle(session, handle: str, httpPrefix: str, storeWebfingerInCache(nickname + '@' + wfDomain, result, cachedWebfingers) else: - print("WARN: Unable to webfinger " + url + ' ' + - 'nickname: ' + str(nickname) + ' ' + - 'domain: ' + str(wfDomain) + ' ' + - 'headers: ' + str(hdr) + ' ' + - 'params: ' + str(par)) + if debug: + print("WARN: Unable to webfinger " + url + ' ' + + 'nickname: ' + str(nickname) + ' ' + + 'domain: ' + str(wfDomain) + ' ' + + 'headers: ' + str(hdr) + ' ' + + 'params: ' + str(par)) return result From e3971c1720e4f59982c10e4b8ab61aed94a46988 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:24:57 +0000 Subject: [PATCH 14/22] Use a file to enable debug mode --- epicyon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/epicyon.py b/epicyon.py index f31ef781a..98f4dc0fe 100644 --- a/epicyon.py +++ b/epicyon.py @@ -514,6 +514,9 @@ args = parser.parse_args() debug = False if args.debug: debug = True +else: + if os.path.isfile('debug'): + debug = True if args.tests: runAllTests() From a4b7b921de6d5124bb9a678dbd46d4ca5fb1728e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:32:11 +0000 Subject: [PATCH 15/22] Less verbose when not in debug --- inbox.py | 7 ++++--- outbox.py | 2 +- posts.py | 11 ++++++----- webapp_post.py | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/inbox.py b/inbox.py index de3587042..c925e19a8 100644 --- a/inbox.py +++ b/inbox.py @@ -414,7 +414,8 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, if isinstance(postJsonObject['object']['content'], str): if isFiltered(baseDir, nickname, domain, postJsonObject['object']['content']): - print('WARN: post was filtered out due to content') + if debug: + print('WARN: post was filtered out due to content') return None originalPostId = None if postJsonObject.get('id'): @@ -1376,7 +1377,7 @@ def _receiveAnnounce(recentPostsCache: {}, __version__, translate, YTReplacementDomain, allowLocalNetworkAccess, - recentPostsCache) + recentPostsCache, debug) if not postJsonObject: notInOnion = True if onionDomain: @@ -2465,7 +2466,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, nickname, domain, postJsonObject, translate, YTReplacementDomain, allowLocalNetworkAccess, - recentPostsCache): + recentPostsCache, debug): # media index will be updated updateIndexList.append('tlmedia') if isBlogPost(postJsonObject): diff --git a/outbox.py b/outbox.py index 05b21240e..7726cf6fb 100644 --- a/outbox.py +++ b/outbox.py @@ -272,7 +272,7 @@ def postMessageToOutbox(session, translate: {}, messageJson, translate, YTReplacementDomain, allowLocalNetworkAccess, - recentPostsCache): + recentPostsCache, debug): inboxUpdateIndex('tlmedia', baseDir, postToNickname + '@' + domain, savedFilename, debug) diff --git a/posts.py b/posts.py index 676ea3e67..b4c09cb15 100644 --- a/posts.py +++ b/posts.py @@ -2927,7 +2927,7 @@ def isImageMedia(session, baseDir: str, httpPrefix: str, postJsonObject: {}, translate: {}, YTReplacementDomain: str, allowLocalNetworkAccess: bool, - recentPostsCache: {}) -> bool: + recentPostsCache: {}, debug: bool) -> bool: """Returns true if the given post has attached image media """ if postJsonObject['type'] == 'Announce': @@ -2937,7 +2937,7 @@ def isImageMedia(session, baseDir: str, httpPrefix: str, __version__, translate, YTReplacementDomain, allowLocalNetworkAccess, - recentPostsCache) + recentPostsCache, debug) if postJsonAnnounce: postJsonObject = postJsonAnnounce if postJsonObject['type'] != 'Create': @@ -3842,7 +3842,7 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, postJsonObject: {}, projectVersion: str, translate: {}, YTReplacementDomain: str, allowLocalNetworkAccess: bool, - recentPostsCache: {}) -> {}: + recentPostsCache: {}, debug: bool) -> {}: """Download the post referenced by an announce """ if not postJsonObject.get('object'): @@ -3866,8 +3866,9 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, return None if os.path.isfile(announceFilename): - print('Reading cached Announce content for ' + - postJsonObject['object']) + if debug: + print('Reading cached Announce content for ' + + postJsonObject['object']) postJsonObject = loadJson(announceFilename) if postJsonObject: return postJsonObject diff --git a/webapp_post.py b/webapp_post.py index 895908213..4b6fdfba3 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1292,7 +1292,7 @@ def individualPostAsHtml(allowDownloads: bool, projectVersion, translate, YTReplacementDomain, allowLocalNetworkAccess, - recentPostsCache) + recentPostsCache, False) if not postJsonAnnounce: # if the announce could not be downloaded then mark it as rejected rejectPostId(baseDir, nickname, domain, postJsonObject['id'], From f074349ab2492b9c6aab81532a2865aadd077017 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:39:00 +0000 Subject: [PATCH 16/22] Less verbose --- daemon.py | 3 ++- inbox.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index d4f4f893a..ce0c2cb35 100644 --- a/daemon.py +++ b/daemon.py @@ -13941,7 +13941,8 @@ class PubServer(BaseHTTPRequestHandler): "editblogpost", "newreminder", "newevent") for currPostType in postTypes: if not authorized: - print('POST was not authorized') + if self.server.debug: + print('POST was not authorized') break postRedirect = self.server.defaultTimeline diff --git a/inbox.py b/inbox.py index c925e19a8..a78e6b709 100644 --- a/inbox.py +++ b/inbox.py @@ -2868,7 +2868,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, time.sleep(1) if not pubKey: - print('Queue: public key could not be obtained from ' + keyId) + if debug: + print('Queue: public key could not be obtained from ' + keyId) if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: From 1a17acd43e0596ae984ce8eea92408b130db5b6f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:42:25 +0000 Subject: [PATCH 17/22] Less verbose --- inbox.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/inbox.py b/inbox.py index a78e6b709..b598cdef6 100644 --- a/inbox.py +++ b/inbox.py @@ -759,8 +759,9 @@ def _personReceiveUpdate(baseDir: str, debug: bool) -> bool: """Changes an actor. eg: avatar or display name change """ - print('Receiving actor update for ' + personJson['url'] + - ' ' + str(personJson)) + if debug: + print('Receiving actor update for ' + personJson['url'] + + ' ' + str(personJson)) domainFull = getFullDomain(domain, port) updateDomainFull = getFullDomain(updateDomain, updatePort) usersPaths = ('users', 'profile', 'channel', 'accounts', 'u') @@ -813,9 +814,10 @@ def _personReceiveUpdate(baseDir: str, # save to cache in memory storePersonInCache(baseDir, personJson['id'], personJson, personCache, True) - # save to cache on file + # save to cache on file if saveJson(personJson, actorFilename): - print('actor updated for ' + personJson['id']) + if debug: + print('actor updated for ' + personJson['id']) # remove avatar if it exists so that it will be refreshed later # when a timeline is constructed @@ -902,7 +904,9 @@ def _receiveUpdate(recentPostsCache: {}, session, baseDir: str, if messageJson['type'] == 'Person': if messageJson.get('url') and messageJson.get('id'): - print('Request to update actor unwrapped: ' + str(messageJson)) + if debug: + print('Request to update actor unwrapped: ' + + str(messageJson)) updateNickname = getNicknameFromActor(messageJson['id']) if updateNickname: updateDomain, updatePort = \ @@ -923,7 +927,8 @@ def _receiveUpdate(recentPostsCache: {}, session, baseDir: str, messageJson['object']['type'] == 'Service': if messageJson['object'].get('url') and \ messageJson['object'].get('id'): - print('Request to update actor: ' + str(messageJson)) + if debug: + print('Request to update actor: ' + str(messageJson)) updateNickname = getNicknameFromActor(messageJson['actor']) if updateNickname: updateDomain, updatePort = \ From b207d2d6745e7d4ac44c461c97a678a6ce51716c Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:46:46 +0000 Subject: [PATCH 18/22] Less verbose --- inbox.py | 5 +++-- posts.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/inbox.py b/inbox.py index b598cdef6..52cae2c1d 100644 --- a/inbox.py +++ b/inbox.py @@ -814,7 +814,7 @@ def _personReceiveUpdate(baseDir: str, # save to cache in memory storePersonInCache(baseDir, personJson['id'], personJson, personCache, True) - # save to cache on file + # save to cache on file if saveJson(personJson, actorFilename): if debug: print('actor updated for ' + personJson['id']) @@ -3048,7 +3048,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, federationList, queueJson['postNickname'], debug): - print('Queue: Update accepted from ' + keyId) + if debug: + print('Queue: Update accepted from ' + keyId) if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: diff --git a/posts.py b/posts.py index b4c09cb15..c088c178d 100644 --- a/posts.py +++ b/posts.py @@ -3908,7 +3908,9 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, print('Announce download blocked object: ' + str(postJsonObject['object'])) return None - print('Downloading Announce content for ' + postJsonObject['object']) + if debug: + print('Downloading Announce content for ' + + postJsonObject['object']) announcedJson = \ getJson(session, postJsonObject['object'], asHeader, None, projectVersion, httpPrefix, domain) From 928412c34cd22886f9b740e777ce87b0ba056593 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 19:53:22 +0000 Subject: [PATCH 19/22] Less verbose --- inbox.py | 6 +++--- newswire.py | 4 ++-- posts.py | 2 +- utils.py | 8 +++++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/inbox.py b/inbox.py index 52cae2c1d..76e7aa55f 100644 --- a/inbox.py +++ b/inbox.py @@ -1608,7 +1608,7 @@ def _estimateNumberOfEmoji(content: str) -> int: def _validPostContent(baseDir: str, nickname: str, domain: str, messageJson: {}, maxMentions: int, maxEmoji: int, - allowLocalNetworkAccess: bool) -> bool: + allowLocalNetworkAccess: bool, debug: bool) -> bool: """Is the content of a received post valid? Check for bad html Check for hellthreads @@ -1627,7 +1627,7 @@ def _validPostContent(baseDir: str, nickname: str, domain: str, return False if 'Z' not in messageJson['object']['published']: return False - if not validPostDate(messageJson['object']['published']): + if not validPostDate(messageJson['object']['published'], 90, debug): return False if messageJson['object'].get('summary'): @@ -2305,7 +2305,7 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, nickname = handle.split('@')[0] if _validPostContent(baseDir, nickname, domain, postJsonObject, maxMentions, maxEmoji, - allowLocalNetworkAccess): + allowLocalNetworkAccess, debug): if postJsonObject.get('object'): jsonObj = postJsonObject['object'] diff --git a/newswire.py b/newswire.py index d54bcfe22..f59ca36cb 100644 --- a/newswire.py +++ b/newswire.py @@ -144,11 +144,11 @@ def _addNewswireDictEntry(baseDir: str, domain: str, ] -def _validFeedDate(pubDate: str) -> bool: +def _validFeedDate(pubDate: str, debug=False) -> bool: # convert from YY-MM-DD HH:MM:SS+00:00 to # YY-MM-DDTHH:MM:SSZ postDate = pubDate.replace(' ', 'T').replace('+00:00', 'Z') - return validPostDate(postDate, 90) + return validPostDate(postDate, 90, debug) def parseFeedDate(pubDate: str) -> str: diff --git a/posts.py b/posts.py index c088c178d..5a6bac82a 100644 --- a/posts.py +++ b/posts.py @@ -3962,7 +3962,7 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, baseDir, nickname, domain, postId, recentPostsCache) return None - if not validPostDate(announcedJson['published']): + if not validPostDate(announcedJson['published'], 90, debug): _rejectAnnounce(announceFilename, baseDir, nickname, domain, postId, recentPostsCache) diff --git a/utils.py b/utils.py index 2d09cf0e0..35d42a4e8 100644 --- a/utils.py +++ b/utils.py @@ -100,7 +100,7 @@ def hasUsersPath(pathStr: str) -> bool: return False -def validPostDate(published: str, maxAgeDays=90) -> bool: +def validPostDate(published: str, maxAgeDays=90, debug=False) -> bool: """Returns true if the published date is recent and is not in the future """ baselineTime = datetime.datetime(1970, 1, 1) @@ -118,11 +118,13 @@ def validPostDate(published: str, maxAgeDays=90) -> bool: postDaysSinceEpoch = daysDiff.days if postDaysSinceEpoch > nowDaysSinceEpoch: - print("Inbox post has a published date in the future!") + if debug: + print("Inbox post has a published date in the future!") return False if nowDaysSinceEpoch - postDaysSinceEpoch >= maxAgeDays: - print("Inbox post is not recent enough") + if debug: + print("Inbox post is not recent enough") return False return True From a3be3c18032f97dfc1269a0a5f385d8508842b4f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 20:15:44 +0000 Subject: [PATCH 20/22] Less verbose --- daemon.py | 5 +++-- inbox.py | 27 ++++++++++++++++----------- session.py | 5 +++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/daemon.py b/daemon.py index ce0c2cb35..92bf42014 100644 --- a/daemon.py +++ b/daemon.py @@ -1333,8 +1333,9 @@ class PubServer(BaseHTTPRequestHandler): return True elif self.path.endswith('/' + nickname): return True - print('AUTH: nickname ' + nickname + - ' was not found in path ' + self.path) + if self.server.debug: + print('AUTH: nickname ' + nickname + + ' was not found in path ' + self.path) return False print('AUTH: epicyon cookie ' + 'authorization failed, header=' + diff --git a/inbox.py b/inbox.py index 76e7aa55f..be2a2420b 100644 --- a/inbox.py +++ b/inbox.py @@ -395,9 +395,10 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, replyDomain, replyPort = \ getDomainFromActor(inReplyTo) if isBlockedDomain(baseDir, replyDomain): - print('WARN: post contains reply from ' + - str(actor) + - ' to a blocked domain: ' + replyDomain) + if debug: + print('WARN: post contains reply from ' + + str(actor) + + ' to a blocked domain: ' + replyDomain) return None else: replyNickname = \ @@ -405,10 +406,11 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, if replyNickname and replyDomain: if isBlocked(baseDir, nickname, domain, replyNickname, replyDomain): - print('WARN: post contains reply from ' + - str(actor) + - ' to a blocked account: ' + - replyNickname + '@' + replyDomain) + if debug: + print('WARN: post contains reply from ' + + str(actor) + + ' to a blocked account: ' + + replyNickname + '@' + replyDomain) return None if postJsonObject['object'].get('content'): if isinstance(postJsonObject['object']['content'], str): @@ -1438,8 +1440,9 @@ def _receiveAnnounce(recentPostsCache: {}, __version__, httpPrefix, domain, onionDomain) if pubKey: - print('DEBUG: public key obtained for announce: ' + - lookupActor) + if debug: + print('DEBUG: public key obtained for announce: ' + + lookupActor) break if debug: @@ -1693,7 +1696,8 @@ def _validPostContent(baseDir: str, nickname: str, domain: str, print('REJECT: reply to post which does not ' + 'allow comments: ' + originalPostId) return False - print('ACCEPT: post content is valid') + if debug: + print('ACCEPT: post content is valid') return True @@ -1735,7 +1739,8 @@ def _obtainAvatarForReplyPost(session, baseDir: str, httpPrefix: str, __version__, httpPrefix, domain, onionDomain) if pubKey: - print('DEBUG: public key obtained for reply: ' + lookupActor) + if debug: + print('DEBUG: public key obtained for reply: ' + lookupActor) break if debug: diff --git a/session.py b/session.py index 879e9bf88..d3cf4437a 100644 --- a/session.py +++ b/session.py @@ -88,8 +88,9 @@ def getJson(session, url: str, headers: {}, params: {}, version='1.2.0', httpPrefix='https', domain='testdomain', timeoutSec=20, quiet=False) -> {}: if not isinstance(url, str): - print('url: ' + str(url)) - print('ERROR: getJson failed, url should be a string') + if not quiet: + print('url: ' + str(url)) + print('ERROR: getJson failed, url should be a string') return None sessionParams = {} sessionHeaders = {} From 041b6f0a8c4ba724792604121f5aced4fd120b44 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 20:41:37 +0000 Subject: [PATCH 21/22] Less verbose --- migrate.py | 2 +- person.py | 2 +- posts.py | 6 +++--- webapp_profile.py | 46 +++++++++++++++++++++++++++------------------- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/migrate.py b/migrate.py index 9c24dce82..383e57971 100644 --- a/migrate.py +++ b/migrate.py @@ -76,7 +76,7 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, 'Accept': 'application/activity+json; profile="' + profileStr + '"' } if not personUrl: - personUrl = getUserUrl(wfRequest) + personUrl = getUserUrl(wfRequest, 0, debug) if not personUrl: return ctr diff --git a/person.py b/person.py index e4d3c3f01..78f82a021 100644 --- a/person.py +++ b/person.py @@ -1204,7 +1204,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, 'Accept': 'application/activity+json; profile="' + profileStr + '"' } if not personUrl: - personUrl = getUserUrl(wfRequest) + personUrl = getUserUrl(wfRequest, 0, debug) if nickname == domain: personUrl = personUrl.replace('/users/', '/actor/') personUrl = personUrl.replace('/accounts/', '/actor/') diff --git a/posts.py b/posts.py index 5a6bac82a..1bfa0dd50 100644 --- a/posts.py +++ b/posts.py @@ -148,7 +148,7 @@ def _cleanHtml(rawHtml: str) -> str: return html.unescape(text) -def getUserUrl(wfRequest: {}, sourceId=0) -> str: +def getUserUrl(wfRequest: {}, sourceId=0, debug=False) -> str: """Gets the actor url from a webfinger request """ if not wfRequest.get('links'): @@ -166,7 +166,7 @@ def getUserUrl(wfRequest: {}, sourceId=0) -> str: if link['type'] != 'application/activity+json': continue if '/@' not in link['href']: - if not hasUsersPath(link['href']): + if debug and not hasUsersPath(link['href']): print('getUserUrl webfinger activity+json ' + 'contains single user instance actor ' + str(sourceId) + ' ' + str(link)) @@ -229,7 +229,7 @@ def getPersonBox(baseDir: str, session, wfRequest: {}, return None, None, None, None, None, None, None if not wfRequest.get('errors'): - personUrl = getUserUrl(wfRequest, sourceId) + personUrl = getUserUrl(wfRequest, sourceId, False) else: if nickname == 'dev': # try single user instance diff --git a/webapp_profile.py b/webapp_profile.py index 0a4902822..2080b438a 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -75,12 +75,14 @@ def htmlProfileAfterSearch(cssCache: {}, searchDomain, searchPort = getDomainFromActor(profileHandle) else: if '@' not in profileHandle: - print('DEBUG: no @ in ' + profileHandle) + if debug: + print('DEBUG: no @ in ' + profileHandle) return None if profileHandle.startswith('@'): profileHandle = profileHandle[1:] if '@' not in profileHandle: - print('DEBUG: no @ in ' + profileHandle) + if debug: + print('DEBUG: no @ in ' + profileHandle) return None searchNickname = profileHandle.split('@')[0] searchDomain = profileHandle.split('@')[1] @@ -91,17 +93,21 @@ def htmlProfileAfterSearch(cssCache: {}, searchPort = int(searchPortStr) searchDomain = searchDomain.split(':')[0] if searchPort: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain) + ':' + - str(searchPort)) + if debug: + print('DEBUG: Search for handle ' + + str(searchNickname) + '@' + str(searchDomain) + ':' + + str(searchPort)) else: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain)) + if debug: + print('DEBUG: Search for handle ' + + str(searchNickname) + '@' + str(searchDomain)) if not searchNickname: - print('DEBUG: No nickname found in ' + profileHandle) + if debug: + print('DEBUG: No nickname found in ' + profileHandle) return None if not searchDomain: - print('DEBUG: No domain found in ' + profileHandle) + if debug: + print('DEBUG: No domain found in ' + profileHandle) return None searchDomainFull = getFullDomain(searchDomain, searchPort) @@ -117,17 +123,19 @@ def htmlProfileAfterSearch(cssCache: {}, httpPrefix, cachedWebfingers, domain, projectVersion, debug) if not wf: - print('DEBUG: Unable to webfinger ' + - searchNickname + '@' + searchDomainFull) - print('DEBUG: cachedWebfingers ' + str(cachedWebfingers)) - print('DEBUG: httpPrefix ' + httpPrefix) - print('DEBUG: domain ' + domain) + if debug: + print('DEBUG: Unable to webfinger ' + + searchNickname + '@' + searchDomainFull) + print('DEBUG: cachedWebfingers ' + str(cachedWebfingers)) + print('DEBUG: httpPrefix ' + httpPrefix) + print('DEBUG: domain ' + domain) return None if not isinstance(wf, dict): - print('WARN: Webfinger search for ' + - searchNickname + '@' + searchDomainFull + - ' did not return a dict. ' + - str(wf)) + if debug: + print('WARN: Webfinger search for ' + + searchNickname + '@' + searchDomainFull + + ' did not return a dict. ' + + str(wf)) return None personUrl = None @@ -140,7 +148,7 @@ def htmlProfileAfterSearch(cssCache: {}, 'Accept': 'application/activity+json; profile="' + profileStr + '"' } if not personUrl: - personUrl = getUserUrl(wf) + personUrl = getUserUrl(wf, 0, debug) if not personUrl: # try single user instance asHeader = { From 6dc85f69dcc4004fe83589e1e4df0e20154999bd Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 14 Mar 2021 20:55:37 +0000 Subject: [PATCH 22/22] Less verbose --- epicyon.py | 2 +- inbox.py | 4 ++-- migrate.py | 4 ++-- person.py | 6 +++--- posts.py | 17 +++++++++-------- session.py | 8 ++++---- speaker.py | 2 +- webapp_profile.py | 4 ++-- webapp_utils.py | 6 +++--- webfinger.py | 3 ++- 10 files changed, 29 insertions(+), 27 deletions(-) diff --git a/epicyon.py b/epicyon.py index 98f4dc0fe..f88e8a30a 100644 --- a/epicyon.py +++ b/epicyon.py @@ -731,7 +731,7 @@ if args.json: 'Accept': 'application/ld+json; profile="' + profileStr + '"' } testJson = getJson(session, args.json, asHeader, None, - __version__, httpPrefix, None) + debug, __version__, httpPrefix, None) pprint(testJson) sys.exit() diff --git a/inbox.py b/inbox.py index be2a2420b..bc9b1183e 100644 --- a/inbox.py +++ b/inbox.py @@ -253,8 +253,8 @@ def getPersonPubKey(baseDir: str, session, personUrl: str, 'Accept': 'application/activity+json; profile="' + profileStr + '"' } personJson = \ - getJson(session, personUrl, asHeader, None, projectVersion, - httpPrefix, personDomain) + getJson(session, personUrl, asHeader, None, debug, + projectVersion, httpPrefix, personDomain) if not personJson: return None pubKey = None diff --git a/migrate.py b/migrate.py index 383e57971..73d7f07a8 100644 --- a/migrate.py +++ b/migrate.py @@ -85,8 +85,8 @@ def _updateMovedHandle(baseDir: str, nickname: str, domain: str, 'Accept': 'application/ld+json; profile="' + profileStr + '"' } personJson = \ - getJson(session, personUrl, asHeader, None, __version__, - httpPrefix, None) + getJson(session, personUrl, asHeader, None, + debug, __version__, httpPrefix, None) if not personJson: return ctr if not personJson.get('movedTo'): diff --git a/person.py b/person.py index 78f82a021..2d6be5a4d 100644 --- a/person.py +++ b/person.py @@ -1225,8 +1225,8 @@ def getActorJson(handle: str, http: bool, gnunet: bool, } personJson = \ - getJson(session, personUrl, asHeader, None, __version__, - httpPrefix, None, 20, quiet) + getJson(session, personUrl, asHeader, None, + debug, __version__, httpPrefix, None, 20, quiet) if personJson: if not quiet: pprint(personJson) @@ -1237,7 +1237,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, } personJson = \ getJson(session, personUrl, asHeader, None, - __version__, httpPrefix, None) + debug, __version__, httpPrefix, None) if not quiet: if personJson: pprint(personJson) diff --git a/posts.py b/posts.py index 1bfa0dd50..7da14e9bd 100644 --- a/posts.py +++ b/posts.py @@ -183,7 +183,7 @@ def parseUserFeed(session, feedUrl: str, asHeader: {}, return None feedJson = getJson(session, feedUrl, asHeader, None, - projectVersion, httpPrefix, domain) + False, projectVersion, httpPrefix, domain) if not feedJson: return None @@ -220,6 +220,7 @@ def getPersonBox(baseDir: str, session, wfRequest: {}, nickname: str, domain: str, boxName='inbox', sourceId=0) -> (str, str, str, str, str, str, str, str): + debug = False profileStr = 'https://www.w3.org/ns/activitystreams' asHeader = { 'Accept': 'application/activity+json; profile="' + profileStr + '"' @@ -229,7 +230,7 @@ def getPersonBox(baseDir: str, session, wfRequest: {}, return None, None, None, None, None, None, None if not wfRequest.get('errors'): - personUrl = getUserUrl(wfRequest, sourceId, False) + personUrl = getUserUrl(wfRequest, sourceId, debug) else: if nickname == 'dev': # try single user instance @@ -250,13 +251,13 @@ def getPersonBox(baseDir: str, session, wfRequest: {}, 'Accept': 'application/ld+json; profile="' + profileStr + '"' } personJson = getJson(session, personUrl, asHeader, None, - projectVersion, httpPrefix, domain) + debug, projectVersion, httpPrefix, domain) if not personJson: asHeader = { 'Accept': 'application/ld+json; profile="' + profileStr + '"' } personJson = getJson(session, personUrl, asHeader, None, - projectVersion, httpPrefix, domain) + debug, projectVersion, httpPrefix, domain) if not personJson: print('Unable to get actor') return None, None, None, None, None, None, None @@ -3537,7 +3538,7 @@ def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str, def downloadFollowCollection(followType: str, session, httpPrefix, actor: str, pageNumber=1, - noOfPages=1) -> []: + noOfPages=1, debug=False) -> []: """Returns a list of following/followers for the given actor by downloading the json for their following/followers collection """ @@ -3554,8 +3555,8 @@ def downloadFollowCollection(followType: str, for pageCtr in range(noOfPages): url = actor + '/' + followType + '?page=' + str(pageNumber + pageCtr) followersJson = \ - getJson(session, url, sessionHeaders, None, __version__, - httpPrefix, None) + getJson(session, url, sessionHeaders, None, + debug, __version__, httpPrefix, None) if followersJson: if followersJson.get('orderedItems'): for followerActor in followersJson['orderedItems']: @@ -3913,7 +3914,7 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, postJsonObject['object']) announcedJson = \ getJson(session, postJsonObject['object'], asHeader, - None, projectVersion, httpPrefix, domain) + None, debug, projectVersion, httpPrefix, domain) if not announcedJson: return None diff --git a/session.py b/session.py index d3cf4437a..f4d6e6dd7 100644 --- a/session.py +++ b/session.py @@ -84,11 +84,11 @@ def urlExists(session, url: str, timeoutSec=3, return False -def getJson(session, url: str, headers: {}, params: {}, +def getJson(session, url: str, headers: {}, params: {}, debug: bool, version='1.2.0', httpPrefix='https', domain='testdomain', timeoutSec=20, quiet=False) -> {}: if not isinstance(url, str): - if not quiet: + if debug and not quiet: print('url: ' + str(url)) print('ERROR: getJson failed, url should be a string') return None @@ -114,7 +114,7 @@ def getJson(session, url: str, headers: {}, params: {}, sessionHeaders2 = sessionHeaders.copy() if sessionHeaders2.get('Authorization'): sessionHeaders2['Authorization'] = 'REDACTED' - if not quiet: + if debug and not quiet: print('ERROR: getJson failed\nurl: ' + str(url) + ' ' + 'headers: ' + str(sessionHeaders2) + ' ' + 'params: ' + str(sessionParams)) @@ -123,7 +123,7 @@ def getJson(session, url: str, headers: {}, params: {}, sessionHeaders2 = sessionHeaders.copy() if sessionHeaders2.get('Authorization'): sessionHeaders2['Authorization'] = 'REDACTED' - if not quiet: + if debug and not quiet: print('ERROR: getJson failed\nurl: ' + str(url) + ' ' + 'headers: ' + str(sessionHeaders2) + ' ' + 'params: ' + str(sessionParams) + ' ') diff --git a/speaker.py b/speaker.py index dd439a17a..8c91b4464 100644 --- a/speaker.py +++ b/speaker.py @@ -279,7 +279,7 @@ def getSpeakerFromServer(baseDir: str, session, domainFull + '/users/' + nickname + '/speaker' speakerJson = \ - getJson(session, url, headers, None, + getJson(session, url, headers, None, debug, __version__, httpPrefix, domain, 20, True) return speakerJson diff --git a/webapp_profile.py b/webapp_profile.py index 2080b438a..0161947e5 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -156,14 +156,14 @@ def htmlProfileAfterSearch(cssCache: {}, } personUrl = httpPrefix + '://' + searchDomainFull profileJson = \ - getJson(session, personUrl, asHeader, None, + getJson(session, personUrl, asHeader, None, debug, projectVersion, httpPrefix, domain) if not profileJson: asHeader = { 'Accept': 'application/ld+json; profile="' + profileStr + '"' } profileJson = \ - getJson(session, personUrl, asHeader, None, + getJson(session, personUrl, asHeader, None, debug, projectVersion, httpPrefix, domain) if not profileJson: print('DEBUG: No actor returned from ' + personUrl) diff --git a/webapp_utils.py b/webapp_utils.py index 0f4fcb761..eca9c1fa0 100644 --- a/webapp_utils.py +++ b/webapp_utils.py @@ -441,7 +441,7 @@ def setBlogAddress(actorJson: {}, blogAddress: str) -> None: def updateAvatarImageCache(session, baseDir: str, httpPrefix: str, actor: str, avatarUrl: str, personCache: {}, allowDownloads: bool, - force=False) -> str: + force=False, debug=False) -> str: """Updates the cached avatar for the given actor """ if not avatarUrl: @@ -502,8 +502,8 @@ def updateAvatarImageCache(session, baseDir: str, httpPrefix: str, 'Accept': 'application/ld+json; profile="' + prof + '"' } personJson = \ - getJson(session, actor, sessionHeaders, None, __version__, - httpPrefix, None) + getJson(session, actor, sessionHeaders, None, + debug, __version__, httpPrefix, None) if personJson: if not personJson.get('id'): return None diff --git a/webfinger.py b/webfinger.py index 920f97af7..7ece61575 100644 --- a/webfinger.py +++ b/webfinger.py @@ -74,7 +74,8 @@ def webfingerHandle(session, handle: str, httpPrefix: str, } try: result = \ - getJson(session, url, hdr, par, projectVersion, + getJson(session, url, hdr, par, + debug, projectVersion, httpPrefix, fromDomain) except Exception as e: print(e)