From a68c76a40f567d279b1be7c89ae58e2f47efb8aa Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 31 May 2021 22:16:51 +0100 Subject: [PATCH 01/20] No referrer policy --- daemon.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/daemon.py b/daemon.py index fb6a3f4f5..1bd188359 100644 --- a/daemon.py +++ b/daemon.py @@ -608,7 +608,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _logout_headers(self, fileFormat: str, length: int, @@ -622,7 +622,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _logout_redirect(self, redirect: str, cookie: str, @@ -639,7 +639,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _set_headers_base(self, fileFormat: str, length: int, cookie: str, @@ -660,7 +660,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') self.send_header('X-Clacks-Overhead', 'GNU Natalie Nguyen') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.send_header('Accept-Ranges', 'none') def _set_headers(self, fileFormat: str, length: int, cookie: str, @@ -747,7 +747,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _httpReturnCode(self, httpCode: int, httpDescription: str, @@ -769,7 +769,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', msgLenStr) self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() if not self._write(msg): print('Error when showing ' + str(httpCode)) From f2954c463f2935a00068f9c72fb60dd6f65607f2 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 31 May 2021 22:31:06 +0100 Subject: [PATCH 02/20] Cache control header --- daemon.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon.py b/daemon.py index 1bd188359..99e9006a4 100644 --- a/daemon.py +++ b/daemon.py @@ -608,6 +608,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -622,6 +623,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -639,6 +641,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -660,6 +663,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') self.send_header('X-Clacks-Overhead', 'GNU Natalie Nguyen') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.send_header('Accept-Ranges', 'none') @@ -747,6 +751,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -769,6 +774,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', msgLenStr) self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') + self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() if not self._write(msg): From a9e6c0a5334e63df832b6046eba51a124856d6f3 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 31 May 2021 22:43:15 +0100 Subject: [PATCH 03/20] Remove cache control --- daemon.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/daemon.py b/daemon.py index 99e9006a4..22f46b75a 100644 --- a/daemon.py +++ b/daemon.py @@ -608,7 +608,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -623,7 +623,7 @@ class PubServer(BaseHTTPRequestHandler): 'title="Login to Epicyon", Basic realm="epicyon"') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -641,7 +641,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -663,14 +663,14 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') self.send_header('X-Clacks-Overhead', 'GNU Natalie Nguyen') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.send_header('Accept-Ranges', 'none') def _set_headers(self, fileFormat: str, length: int, cookie: str, callingDomain: str) -> None: self._set_headers_base(fileFormat, length, cookie, callingDomain) - self.send_header('Cache-Control', 'public, max-age=0') + # self.send_header('Cache-Control', 'public, max-age=0') self.end_headers() def _set_headers_head(self, fileFormat: str, length: int, etag: str, @@ -684,7 +684,7 @@ class PubServer(BaseHTTPRequestHandler): data, cookie: str, callingDomain: str) -> None: datalen = len(data) self._set_headers_base(fileFormat, datalen, cookie, callingDomain) - self.send_header('Cache-Control', 'public, max-age=86400') + # self.send_header('Cache-Control', 'public, max-age=86400') etag = None if os.path.isfile(mediaFilename + '.etag'): try: @@ -751,7 +751,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', '0') self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -774,7 +774,7 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Length', msgLenStr) self.send_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Cache-Control', 'public') + # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() if not self._write(msg): From 1d3ff5f628a10551bb1b5694a99ea5f92df52380 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 31 May 2021 22:51:30 +0100 Subject: [PATCH 04/20] Remove X-robots --- daemon.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/daemon.py b/daemon.py index 22f46b75a..7ad7d5ee4 100644 --- a/daemon.py +++ b/daemon.py @@ -606,8 +606,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('WWW-Authenticate', 'title="Login to Epicyon", Basic realm="epicyon"') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -621,8 +621,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('WWW-Authenticate', 'title="Login to Epicyon", Basic realm="epicyon"') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -639,8 +639,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) self.send_header('Content-Length', '0') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -660,8 +660,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Cookie', cookieStr) self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') self.send_header('X-Clacks-Overhead', 'GNU Natalie Nguyen') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') @@ -749,8 +749,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) self.send_header('Content-Length', '0') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() @@ -772,8 +772,8 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Type', 'text/html; charset=utf-8') msgLenStr = str(len(msg)) self.send_header('Content-Length', msgLenStr) - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') # self.send_header('Cache-Control', 'public') # self.send_header('Referrer-Policy', 'origin') self.end_headers() From d4a6a2437b43eea19abfedb10baa42184a3db919 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 20:06:30 +0100 Subject: [PATCH 05/20] Duplicate definitions --- person.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/person.py b/person.py index 84d0af727..46affd364 100644 --- a/person.py +++ b/person.py @@ -1323,12 +1323,10 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if not personUrl: # try single user instance personUrl = httpPrefix + '://' + domain - profileStr = 'https://www.w3.org/ns/activitystreams' asHeader = { 'Accept': 'application/ld+json; profile="' + profileStr + '"' } if '/channel/' in personUrl or '/accounts/' in personUrl: - profileStr = 'https://www.w3.org/ns/activitystreams' asHeader = { 'Accept': 'application/ld+json; profile="' + profileStr + '"' } @@ -1341,7 +1339,6 @@ def getActorJson(handle: str, http: bool, gnunet: bool, pprint(personJson) return personJson else: - profileStr = 'https://www.w3.org/ns/activitystreams' asHeader = { 'Accept': 'application/jrd+json; profile="' + profileStr + '"' } From 0f81cc6ef895369df9d5bdc44d99602255db7dc0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 20:38:12 +0100 Subject: [PATCH 06/20] Tidying --- person.py | 42 +++++++++++++++++------------------------- utils.py | 1 - 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/person.py b/person.py index 46affd364..01988602d 100644 --- a/person.py +++ b/person.py @@ -1309,9 +1309,9 @@ def getActorJson(handle: str, http: bool, gnunet: bool, return None profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/activity+json; profile="' + profileStr + '"' - } + headersList = ( + "activity+json", "ld+json", "jrd+json" + ) if not personUrl: personUrl = getUserUrl(wfRequest, 0, debug) if nickname == domain: @@ -1323,32 +1323,24 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if not personUrl: # try single user instance personUrl = httpPrefix + '://' + domain - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } + headersList = ( + "ld+json", "jrd+json", "activity+json" + ) if '/channel/' in personUrl or '/accounts/' in personUrl: - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } + headersList = ( + "ld+json", "jrd+json", "activity+json" + ) - personJson = \ - getJson(session, personUrl, asHeader, None, - debug, __version__, httpPrefix, None, 20, quiet) - if personJson: - if not quiet: - pprint(personJson) - return personJson - else: + for headerType in headersList: + headerMimeType = 'application/' + headerType asHeader = { - 'Accept': 'application/jrd+json; profile="' + profileStr + '"' + 'Accept': headerMimeType + '; profile="' + profileStr + '"' } personJson = \ getJson(session, personUrl, asHeader, None, - debug, __version__, httpPrefix, None) - if not quiet: - if personJson: - print('getActorJson returned actor') + debug, __version__, httpPrefix, None, 20, quiet) + if personJson: + if not quiet: pprint(personJson) - else: - print('Failed to get ' + personUrl) - return personJson + return personJson + return None diff --git a/utils.py b/utils.py index 148199f68..79e1cf1a3 100644 --- a/utils.py +++ b/utils.py @@ -15,7 +15,6 @@ import json import idna import locale from pprint import pprint -from calendar import monthrange from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes From e44d1b5e26dedcad277a1126836c1fa83c2c1610 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:04:07 +0100 Subject: [PATCH 07/20] Count the number of posts --- metadata.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/metadata.py b/metadata.py index e073692c3..14ff958c5 100644 --- a/metadata.py +++ b/metadata.py @@ -60,6 +60,22 @@ def metaDataNodeInfo(baseDir: str, return nodeinfo +def _getStatusCount(baseDir: str) -> int: + """Get the total number of posts + """ + statusCtr = 0 + accountsDir = baseDir + '/accounts' + for subdir, dirs, files in os.walk(accountsDir): + for acct in dirs: + if '@' not in acct: + continue + if 'inbox@' in acct or 'news@' in acct: + continue + statusCtr += len(os.path.join(accountsDir, acct)) + break + return statusCtr + + def metaDataInstance(instanceTitle: str, instanceDescriptionShort: str, instanceDescription: str, @@ -117,7 +133,7 @@ def metaDataInstance(instanceTitle: str, 'short_description': instanceDescriptionShort, 'stats': { 'domain_count': 2, - 'status_count': 1, + 'status_count': _getStatusCount(baseDir), 'user_count': noOfAccounts(baseDir) }, 'thumbnail': httpPrefix + '://' + domainFull + '/login.png', From 47fa228cf452efb42b45009c2818cc2a1699e6bf Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:07:29 +0100 Subject: [PATCH 08/20] Outbox --- metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.py b/metadata.py index 14ff958c5..c05aeb49e 100644 --- a/metadata.py +++ b/metadata.py @@ -71,7 +71,7 @@ def _getStatusCount(baseDir: str) -> int: continue if 'inbox@' in acct or 'news@' in acct: continue - statusCtr += len(os.path.join(accountsDir, acct)) + statusCtr += len(os.path.join(accountsDir, acct + '/outbox')) break return statusCtr From f511fccecd059583adf9cc167ec67ec1ed896fb1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:09:35 +0100 Subject: [PATCH 09/20] Remove status count for contact --- metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata.py b/metadata.py index c05aeb49e..f1768b8ad 100644 --- a/metadata.py +++ b/metadata.py @@ -122,7 +122,6 @@ def metaDataInstance(instanceTitle: str, 'last_status_at': '2019-07-01T10:30:00Z', 'locked': adminActor['manuallyApprovesFollowers'], 'note': '

Admin of ' + domain + '

', - 'statuses_count': 1, 'url': url, 'username': adminActor['preferredUsername'] }, From 3a1819e751ff5e90ac126b61e3aaeefcc40c5e0b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:12:19 +0100 Subject: [PATCH 10/20] Remove unused instance fields --- metadata.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/metadata.py b/metadata.py index f1768b8ad..dd3659fd1 100644 --- a/metadata.py +++ b/metadata.py @@ -110,16 +110,9 @@ def metaDataInstance(instanceTitle: str, 'avatar': adminActor['icon']['url'], 'avatar_static': adminActor['icon']['url'], 'bot': isBot, - 'created_at': '2019-07-01T10:30:00Z', 'display_name': adminActor['name'], - 'emojis': [], - 'fields': [], - 'followers_count': 1, - 'following_count': 1, 'header': adminActor['image']['url'], 'header_static': adminActor['image']['url'], - 'id': '1', - 'last_status_at': '2019-07-01T10:30:00Z', 'locked': adminActor['manuallyApprovesFollowers'], 'note': '

Admin of ' + domain + '

', 'url': url, From 190bcfc86953aff08c777eeae3e9dac4a14efa66 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:17:47 +0100 Subject: [PATCH 11/20] Remove email from instance metadata --- metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata.py b/metadata.py index dd3659fd1..5687b465d 100644 --- a/metadata.py +++ b/metadata.py @@ -119,7 +119,6 @@ def metaDataInstance(instanceTitle: str, 'username': adminActor['preferredUsername'] }, 'description': instanceDescription, - 'email': 'admin@' + domain, 'languages': [systemLanguage], 'registrations': registration, 'short_description': instanceDescriptionShort, From a608bc0fffd2f567526fc4463b3f5ad7feb21d36 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:25:39 +0100 Subject: [PATCH 12/20] Count files in directory --- metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata.py b/metadata.py index 5687b465d..48b106ed5 100644 --- a/metadata.py +++ b/metadata.py @@ -71,7 +71,8 @@ def _getStatusCount(baseDir: str) -> int: continue if 'inbox@' in acct or 'news@' in acct: continue - statusCtr += len(os.path.join(accountsDir, acct + '/outbox')) + for f in os.path.join(accountsDir, acct + '/outbox'): + statusCtr += 1 break return statusCtr From b790b2a64b8114f68191df9451e7a2e5109e79a4 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:29:08 +0100 Subject: [PATCH 13/20] Count files in directory --- metadata.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metadata.py b/metadata.py index 48b106ed5..3be09b864 100644 --- a/metadata.py +++ b/metadata.py @@ -71,8 +71,10 @@ def _getStatusCount(baseDir: str) -> int: continue if 'inbox@' in acct or 'news@' in acct: continue - for f in os.path.join(accountsDir, acct + '/outbox'): - statusCtr += 1 + acctDir = os.path.join(accountsDir, acct + '/outbox') + for subdir2, dirs2, files2 in os.walk(acctDir): + statusCtr += len(files) + break break return statusCtr From 042664e9f8af2e80e07a4936779de4b58291222e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 2 Jun 2021 23:30:18 +0100 Subject: [PATCH 14/20] Count files in directory --- metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.py b/metadata.py index 3be09b864..1c117a672 100644 --- a/metadata.py +++ b/metadata.py @@ -73,7 +73,7 @@ def _getStatusCount(baseDir: str) -> int: continue acctDir = os.path.join(accountsDir, acct + '/outbox') for subdir2, dirs2, files2 in os.walk(acctDir): - statusCtr += len(files) + statusCtr += len(files2) break break return statusCtr From 613e6b8945caa2c8f587b62177fae5b4441b182e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 09:33:50 +0100 Subject: [PATCH 15/20] Reject self-boosts These are easily abused by spammers --- announce.py | 4 ++++ inbox.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/announce.py b/announce.py index 8c8486a98..4e254f7f8 100644 --- a/announce.py +++ b/announce.py @@ -32,6 +32,8 @@ def outboxAnnounce(recentPostsCache: {}, """ if not messageJson.get('actor'): return False + if not isinstance(messageJson['actor'], str): + return False if not messageJson.get('type'): return False if not messageJson.get('object'): @@ -39,6 +41,8 @@ def outboxAnnounce(recentPostsCache: {}, if messageJson['type'] == 'Announce': if not isinstance(messageJson['object'], str): return False + if messageJson['actor'] in messageJson['object']: + return False nickname = getNicknameFromActor(messageJson['actor']) if not nickname: print('WARN: no nickname found in ' + messageJson['actor']) diff --git a/inbox.py b/inbox.py index 38978dd4e..2ccb07316 100644 --- a/inbox.py +++ b/inbox.py @@ -1359,6 +1359,10 @@ def _receiveAnnounce(recentPostsCache: {}, '"users" or "profile" missing from actor in ' + messageJson['type']) return False + if messageJson['actor'] in messageJson['object']: + if debug: + print('DEBUG: self-boost rejected') + return False if not hasUsersPath(messageJson['object']): if debug: print('DEBUG: ' + From ea6571da4e3324cc1b946dbbce32eedb6fc2ce91 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 13:17:24 +0100 Subject: [PATCH 16/20] Don't download self-boosts --- posts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/posts.py b/posts.py index 1d4801d78..726357b59 100644 --- a/posts.py +++ b/posts.py @@ -3924,6 +3924,9 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, return None if not isinstance(postJsonObject['object'], str): return None + # ignore self-boosts + if postJsonObject['actor'] in postJsonObject['object']: + return None # get the announced post announceCacheDir = baseDir + '/cache/announce/' + nickname From efe1b54b38280dff15c2e190d796cee3a04964aa Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 14:21:57 +0100 Subject: [PATCH 17/20] Checking for self-announces using a function --- announce.py | 20 +++++++++++++++++++- inbox.py | 3 ++- webapp_timeline.py | 3 +++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/announce.py b/announce.py index 4e254f7f8..dccfb20ea 100644 --- a/announce.py +++ b/announce.py @@ -41,7 +41,7 @@ def outboxAnnounce(recentPostsCache: {}, if messageJson['type'] == 'Announce': if not isinstance(messageJson['object'], str): return False - if messageJson['actor'] in messageJson['object']: + if isSelfAnnounce(messageJson): return False nickname = getNicknameFromActor(messageJson['actor']) if not nickname: @@ -390,3 +390,21 @@ def outboxUndoAnnounce(recentPostsCache: {}, messageJson['actor'], domain, debug) if debug: print('DEBUG: post undo announce via c2s - ' + postFilename) + + +def isSelfAnnounce(postJsonObject: {}) -> bool: + """Is the given post a self announce? + """ + if not postJsonObject.get('actor'): + return False + if not postJsonObject.get('type'): + return False + if postJsonObject['type'] != 'Announce': + return False + if not postJsonObject.get('object'): + return False + if not isinstance(postJsonObject['actor'], str): + return False + if not isinstance(postJsonObject['object'], str): + return False + return postJsonObject['actor'] in postJsonObject['object'] diff --git a/inbox.py b/inbox.py index 2ccb07316..05ba550e9 100644 --- a/inbox.py +++ b/inbox.py @@ -80,6 +80,7 @@ from delete import removeOldHashtags from categories import guessHashtagCategory from context import hasValidContext from speaker import updateSpeaker +from announce import isSelfAnnounce def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -1359,7 +1360,7 @@ def _receiveAnnounce(recentPostsCache: {}, '"users" or "profile" missing from actor in ' + messageJson['type']) return False - if messageJson['actor'] in messageJson['object']: + if isSelfAnnounce(messageJson): if debug: print('DEBUG: self-boost rejected') return False diff --git a/webapp_timeline.py b/webapp_timeline.py index 9500ffb6e..50ec00fbc 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -31,6 +31,7 @@ from webapp_column_left import getLeftColumnContent from webapp_column_right import getRightColumnContent from webapp_headerbuttons import headerButtonsTimeline from posts import isModerator +from announce import isSelfAnnounce def _logTimelineTiming(enableTimingLog: bool, timelineStartTime, @@ -684,6 +685,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, # is the actor who sent this post snoozed? if isPersonSnoozed(baseDir, nickname, domain, item['actor']): continue + if isSelfAnnounce(item): + continue # is the post in the memory cache of recent ones? currTlStr = None From ca17ba9fe3d53c8f48c867522b01adfe5179686b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 19:30:48 +0100 Subject: [PATCH 18/20] Additional users path --- person.py | 20 +++++++++++++++----- utils.py | 10 ++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/person.py b/person.py index 01988602d..d0927ba7a 100644 --- a/person.py +++ b/person.py @@ -1216,15 +1216,16 @@ def getActorJson(handle: str, http: bool, gnunet: bool, handle.startswith('http') or \ handle.startswith('dat'): # format: https://domain/@nick - prefixes = getProtocolPrefixes() - for prefix in prefixes: - handle = handle.replace(prefix, '') - handle = handle.replace('/@', '/users/') - if not hasUsersPath(handle): + originalHandle = handle + if not hasUsersPath(originalHandle): if not quiet or debug: print('getActorJson: Expected actor format: ' + 'https://domain/@nick or https://domain/users/nick') return None + prefixes = getProtocolPrefixes() + for prefix in prefixes: + handle = handle.replace(prefix, '') + handle = handle.replace('/@', '/users/') if '/users/' in handle: nickname = handle.split('/users/')[1] nickname = nickname.replace('\n', '').replace('\r', '') @@ -1245,6 +1246,15 @@ def getActorJson(handle: str, http: bool, gnunet: bool, nickname = handle.split('/u/')[1] nickname = nickname.replace('\n', '').replace('\r', '') domain = handle.split('/u/')[0] + elif '://' in originalHandle: + domain = originalHandle.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in originalHandle: + return None + nickname = originalHandle.split('://' + domain + '/')[1] + if '/' in nickname or '.' in nickname: + return None else: # format: @nick@domain if '@' not in handle: diff --git a/utils.py b/utils.py index 79e1cf1a3..98e0dd309 100644 --- a/utils.py +++ b/utils.py @@ -97,6 +97,16 @@ def hasUsersPath(pathStr: str) -> bool: for usersStr in usersList: if '/' + usersStr + '/' in pathStr: return True + if '://' in pathStr: + domain = pathStr.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in pathStr: + return False + nickname = pathStr.split('://' + domain + '/')[1] + if '/' in nickname or '.' in nickname: + return False + return True return False From 1c732f5d705b5203a11beb77e0b7a4bcfe1a0a00 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 19:49:09 +0100 Subject: [PATCH 19/20] Resolving smithereen actors --- person.py | 5 +++-- utils.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/person.py b/person.py index d0927ba7a..f3386a658 100644 --- a/person.py +++ b/person.py @@ -1332,7 +1332,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, personUrl = personUrl.replace('/u/', '/actor/') if not personUrl: # try single user instance - personUrl = httpPrefix + '://' + domain + personUrl = httpPrefix + '://' + domain + '/' + nickname headersList = ( "ld+json", "jrd+json", "activity+json" ) @@ -1340,7 +1340,8 @@ def getActorJson(handle: str, http: bool, gnunet: bool, headersList = ( "ld+json", "jrd+json", "activity+json" ) - + if debug: + print('personUrl: ' + personUrl) for headerType in headersList: headerMimeType = 'application/' + headerType asHeader = { diff --git a/utils.py b/utils.py index 98e0dd309..428a6c5f3 100644 --- a/utils.py +++ b/utils.py @@ -851,6 +851,16 @@ def getNicknameFromActor(actor: str) -> str: elif '@' in actor: nickStr = actor.split('@')[0] return nickStr + elif '://' in actor: + domain = actor.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in actor: + return None + nickStr = actor.split('://' + domain + '/')[1] + if '/' in nickStr or '.' in nickStr: + return None + return nickStr return None nickStr = actor.split('/users/')[1].replace('@', '') if '/' not in nickStr: From 73da024042008da5fbf64973224cf6a1dc9f1ac3 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 3 Jun 2021 20:46:35 +0100 Subject: [PATCH 20/20] Remove duplicate actor search --- desktop_client.py | 4 +- person.py | 22 +++++----- pgp.py | 4 +- webapp_profile.py | 105 ++++++---------------------------------------- 4 files changed, 28 insertions(+), 107 deletions(-) diff --git a/desktop_client.py b/desktop_client.py index de1ad92b0..caa39516d 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -836,7 +836,7 @@ def _desktopShowProfile(session, nickname: str, domain: str, isHttp = False if 'http://' in actor: isHttp = True - actorJson = getActorJson(actor, isHttp, False, False, True) + actorJson, asHeader = getActorJson(actor, isHttp, False, False, True) _desktopShowActor(baseDir, actorJson, translate, systemLanguage, screenreader, espeak) @@ -854,7 +854,7 @@ def _desktopShowProfileFromHandle(session, nickname: str, domain: str, """Shows the profile for a handle Returns the actor json """ - actorJson = getActorJson(handle, False, False, False, True) + actorJson, asHeader = getActorJson(handle, False, False, False, True) _desktopShowActor(baseDir, actorJson, translate, systemLanguage, screenreader, espeak) diff --git a/person.py b/person.py index f3386a658..9dea15d95 100644 --- a/person.py +++ b/person.py @@ -1205,7 +1205,7 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, def getActorJson(handle: str, http: bool, gnunet: bool, - debug: bool, quiet=False) -> {}: + debug: bool, quiet=False) -> ({}, {}): """Returns the actor json """ if debug: @@ -1221,7 +1221,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if not quiet or debug: print('getActorJson: Expected actor format: ' + 'https://domain/@nick or https://domain/users/nick') - return None + return None, None prefixes = getProtocolPrefixes() for prefix in prefixes: handle = handle.replace(prefix, '') @@ -1251,22 +1251,22 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if '/' in domain: domain = domain.split('/')[0] if '://' + domain + '/' not in originalHandle: - return None + return None, None nickname = originalHandle.split('://' + domain + '/')[1] if '/' in nickname or '.' in nickname: - return None + return None, None else: # format: @nick@domain if '@' not in handle: if not quiet: print('getActorJson Syntax: --actor nickname@domain') - return None + return None, None if handle.startswith('@'): handle = handle[1:] if '@' not in handle: if not quiet: print('getActorJsonSyntax: --actor nickname@domain') - return None + return None, None nickname = handle.split('@')[0] domain = handle.split('@')[1] domain = domain.replace('\n', '').replace('\r', '') @@ -1297,12 +1297,12 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if not wfRequest: if not quiet: print('getActorJson Unable to webfinger ' + handle) - return None + return None, None if not isinstance(wfRequest, dict): if not quiet: print('getActorJson Webfinger for ' + handle + ' did not return a dict. ' + str(wfRequest)) - return None + return None, None if not quiet: pprint(wfRequest) @@ -1316,7 +1316,7 @@ def getActorJson(handle: str, http: bool, gnunet: bool, else: if debug: print('No users path in ' + handle) - return None + return None, None profileStr = 'https://www.w3.org/ns/activitystreams' headersList = ( @@ -1353,5 +1353,5 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if personJson: if not quiet: pprint(personJson) - return personJson - return None + return personJson, asHeader + return None, None diff --git a/pgp.py b/pgp.py index 81720f3a3..722adb707 100644 --- a/pgp.py +++ b/pgp.py @@ -336,7 +336,7 @@ def _getPGPPublicKeyFromActor(handle: str, actorJson=None) -> str: public key specified """ if not actorJson: - actorJson = getActorJson(handle, False, False, False, True) + actorJson, asHeader = getActorJson(handle, False, False, False, True) if not actorJson: return None if not actorJson.get('attachment'): @@ -476,7 +476,7 @@ def pgpPublicKeyUpload(baseDir: str, session, if debug: print('Getting actor for ' + handle) - actorJson = getActorJson(handle, False, False, debug, True) + actorJson, asHeader = getActorJson(handle, False, False, debug, True) if not actorJson: if debug: print('No actor returned for ' + handle) diff --git a/webapp_profile.py b/webapp_profile.py index b97d0e9a9..00ff979dc 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -10,7 +10,6 @@ import os from pprint import pprint from utils import getOccupationName from utils import getLockedAccount -from utils import hasUsersPath from utils import getFullDomain from utils import isArtist from utils import isDormant @@ -24,10 +23,9 @@ from utils import getImageFormats from skills import getSkills from theme import getThemesList from person import personBoxJson +from person import getActorJson from webfinger import webfingerHandle -from session import getJson from posts import parseUserFeed -from posts import getUserUrl from posts import getPersonBox from donate import getDonationUrl from xmpp import getXmppAddress @@ -74,46 +72,20 @@ def htmlProfileAfterSearch(cssCache: {}, accessKeys: {}) -> str: """Show a profile page after a search for a fediverse address """ - if hasUsersPath(profileHandle) or '/@' in profileHandle: - searchNickname = getNicknameFromActor(profileHandle) - searchDomain, searchPort = getDomainFromActor(profileHandle) - else: - if '@' not in profileHandle: - if debug: - print('DEBUG: no @ in ' + profileHandle) - return None - if profileHandle.startswith('@'): - profileHandle = profileHandle[1:] - if '@' not in profileHandle: - if debug: - print('DEBUG: no @ in ' + profileHandle) - return None - searchNickname = profileHandle.split('@')[0] - searchDomain = profileHandle.split('@')[1] - searchPort = None - if ':' in searchDomain: - searchPortStr = searchDomain.split(':')[1] - if searchPortStr.isdigit(): - searchPort = int(searchPortStr) - searchDomain = searchDomain.split(':')[0] - if searchPort: - if debug: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain) + ':' + - str(searchPort)) - else: - if debug: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain)) - if not searchNickname: - if debug: - print('DEBUG: No nickname found in ' + profileHandle) - return None - if not searchDomain: - if debug: - print('DEBUG: No domain found in ' + profileHandle) + http = False + gnunet = False + if httpPrefix == 'http': + http = True + elif httpPrefix == 'gnunet': + gnunet = True + profileJson, asHeader = \ + getActorJson(profileHandle, http, gnunet, debug, False) + if not profileJson: return None + personUrl = profileJson['id'] + searchDomain, searchPort = getDomainFromActor(personUrl) + searchNickname = getNicknameFromActor(personUrl) searchDomainFull = getFullDomain(searchDomain, searchPort) profileStr = '' @@ -121,57 +93,6 @@ def htmlProfileAfterSearch(cssCache: {}, if os.path.isfile(baseDir + '/epicyon.css'): cssFilename = baseDir + '/epicyon.css' - wf = \ - webfingerHandle(session, - searchNickname + '@' + searchDomainFull, - httpPrefix, cachedWebfingers, - domain, projectVersion, debug) - if not wf: - if debug: - print('DEBUG: Unable to webfinger ' + - searchNickname + '@' + searchDomainFull) - print('DEBUG: cachedWebfingers ' + str(cachedWebfingers)) - print('DEBUG: httpPrefix ' + httpPrefix) - print('DEBUG: domain ' + domain) - return None - if not isinstance(wf, dict): - if debug: - print('WARN: Webfinger search for ' + - searchNickname + '@' + searchDomainFull + - ' did not return a dict. ' + - str(wf)) - return None - - personUrl = None - if wf.get('errors'): - personUrl = httpPrefix + '://' + \ - searchDomainFull + '/users/' + searchNickname - - profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/activity+json; profile="' + profileStr + '"' - } - if not personUrl: - personUrl = getUserUrl(wf, 0, debug) - if not personUrl: - # try single user instance - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } - personUrl = httpPrefix + '://' + searchDomainFull - profileJson = \ - getJson(session, personUrl, asHeader, None, debug, - projectVersion, httpPrefix, domain) - if not profileJson: - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } - profileJson = \ - getJson(session, personUrl, asHeader, None, debug, - projectVersion, httpPrefix, domain) - if not profileJson: - print('DEBUG: No actor returned from ' + personUrl) - return None avatarUrl = '' if profileJson.get('icon'): if profileJson['icon'].get('url'):