From db7043dfd2c7a13bca8621e88900f92ddca4d9c2 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Fri, 1 Jan 2021 23:36:48 +0000 Subject: [PATCH 01/28] Increment year --- epicyon-notification | 2 +- epicyon.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/epicyon-notification b/epicyon-notification index 0f7bd8fde..03043f978 100755 --- a/epicyon-notification +++ b/epicyon-notification @@ -11,7 +11,7 @@ # License # ======= # -# Copyright (C) 2020 Bob Mottram +# Copyright (C) 2020-2021 Bob Mottram # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by diff --git a/epicyon.py b/epicyon.py index a8d25be4b..ec0d9c843 100644 --- a/epicyon.py +++ b/epicyon.py @@ -478,7 +478,6 @@ if args.debug: if args.tests: runAllTests() sys.exit() - if args.testsnetwork: print('Network Tests') testPostMessageBetweenServers() From 2bbdbe97742f5196fa9575024f2d5ccbe054dd2f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Jan 2021 10:37:19 +0000 Subject: [PATCH 02/28] Add ftp as a supported link type --- tests.py | 27 +++++++++++++++++++++++++++ utils.py | 6 ++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 7fcdccd03..b9bac2492 100644 --- a/tests.py +++ b/tests.py @@ -2978,9 +2978,36 @@ def testFunctions(): '-Gsep=+120 -Tx11 epicyon.dot') +def testLinksWithinPost() -> None: + baseDir = os.getcwd() + nickname = 'test27636' + domain = 'rando.site' + port = 443 + httpPrefix = 'https' + content = 'This is a test post with links.\n\n' + \ + 'ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/v4/\n\nhttps://freedombone.net' + postJsonObject = \ + createPublicPost(baseDir, nickname, domain, port, httpPrefix, + content, + False, False, False, True, + None, None, False, None) + assert postJsonObject['object']['content'] == \ + '

This is a test post with links.

' + \ + '' + \ + '' + \ + '' + \ + 'ftp.ncdc.noaa.gov/pub/data/ghcn/v4/' + \ + '

' + \ + '' + \ + 'freedombone.net

' + + def runAllTests(): print('Running tests...') testFunctions() + testLinksWithinPost() testReplyToPublicPost() testGetMentionedPeople() testGuessHashtagCategory() diff --git a/utils.py b/utils.py index 81682d1e1..ad3e2eddf 100644 --- a/utils.py +++ b/utils.py @@ -328,14 +328,16 @@ def removeIdEnding(idStr: str) -> str: def getProtocolPrefixes() -> []: """Returns a list of valid prefixes """ - return ('https://', 'http://', 'dat://', 'i2p://', 'gnunet://', + return ('https://', 'http://', 'ftp://', + 'dat://', 'i2p://', 'gnunet://', 'hyper://', 'gemini://', 'gopher://') def getLinkPrefixes() -> []: """Returns a list of valid web link prefixes """ - return ('https://', 'http://', 'dat://', 'i2p://', 'gnunet://', + return ('https://', 'http://', 'ftp://', + 'dat://', 'i2p://', 'gnunet://', 'hyper://', 'gemini://', 'gopher://', 'briar:') From 74547ca8d9ed1ee6171f629388d96e8c608a650d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Jan 2021 11:06:08 +0000 Subject: [PATCH 03/28] Show locked account status on person options --- daemon.py | 6 +++++- person.py | 10 ++++++++++ webapp_person_options.py | 5 ++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index eefc8965b..9f471794c 100644 --- a/daemon.py +++ b/daemon.py @@ -47,6 +47,7 @@ from matrix import getMatrixAddress from matrix import setMatrixAddress from donate import getDonationUrl from donate import setDonationUrl +from person import getLockedAccount from person import setPersonNotes from person import getDefaultPersonContext from person import savePersonQrcode @@ -5214,11 +5215,13 @@ class PubServer(BaseHTTPRequestHandler): jamiAddress = None ssbAddress = None emailAddress = None + lockedAccount = False actorJson = getPersonFromCache(baseDir, optionsActor, self.server.personCache, True) if actorJson: + lockedAccount = getLockedAccount(actorJson) donateUrl = getDonationUrl(actorJson) xmppAddress = getXmppAddress(actorJson) matrixAddress = getMatrixAddress(actorJson) @@ -5247,7 +5250,8 @@ class PubServer(BaseHTTPRequestHandler): PGPpubKey, PGPfingerprint, emailAddress, self.server.dormantMonths, - backToPath).encode('utf-8') + backToPath, + lockedAccount).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, cookie, callingDomain) diff --git a/person.py b/person.py index b7e80061a..8cf6a0511 100644 --- a/person.py +++ b/person.py @@ -201,6 +201,16 @@ def getDefaultPersonContext() -> str: } +def getLockedAccount(actorJson: {}) -> bool: + """Returns whether the given account requires follower approval + """ + if not actorJson.get('manuallyApprovesFollowers'): + return False + if actorJson['manuallyApprovesFollowers'] is True: + return True + return False + + def _createPersonBase(baseDir: str, nickname: str, domain: str, port: int, httpPrefix: str, saveToFile: bool, manualFollowerApproval: bool, diff --git a/webapp_person_options.py b/webapp_person_options.py index cf6ecadbc..60ecc2eff 100644 --- a/webapp_person_options.py +++ b/webapp_person_options.py @@ -45,7 +45,8 @@ def htmlPersonOptions(defaultTimeline: str, PGPfingerprint: str, emailAddress: str, dormantMonths: int, - backToPath: str) -> str: + backToPath: str, + lockedAccount: bool) -> str: """Show options for a person: view/follow/block/report """ optionsDomain, optionsPort = getDomainFromActor(optionsActor) @@ -112,6 +113,8 @@ def htmlPersonOptions(defaultTimeline: str, '" ' + getBrokenLinkSubstitute() + '/>\n' handle = getNicknameFromActor(optionsActor) + '@' + optionsDomain handleShown = handle + if lockedAccount: + handleShown += '🔒' if dormant: handleShown += ' 💤' optionsStr += \ From a9fcabd366adfc5ec4a11e8b614527e4476e8a3d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Jan 2021 11:18:43 +0000 Subject: [PATCH 04/28] Show locked account status after search for handle --- daemon.py | 2 +- person.py | 10 ---------- utils.py | 10 ++++++++++ webapp_profile.py | 6 ++++++ 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/daemon.py b/daemon.py index 9f471794c..2d1ec1721 100644 --- a/daemon.py +++ b/daemon.py @@ -47,7 +47,6 @@ from matrix import getMatrixAddress from matrix import setMatrixAddress from donate import getDonationUrl from donate import setDonationUrl -from person import getLockedAccount from person import setPersonNotes from person import getDefaultPersonContext from person import savePersonQrcode @@ -178,6 +177,7 @@ from shares import addShare from shares import removeShare from shares import expireShares from categories import setHashtagCategory +from utils import getLockedAccount from utils import hasUsersPath from utils import getFullDomain from utils import removeHtml diff --git a/person.py b/person.py index 8cf6a0511..b7e80061a 100644 --- a/person.py +++ b/person.py @@ -201,16 +201,6 @@ def getDefaultPersonContext() -> str: } -def getLockedAccount(actorJson: {}) -> bool: - """Returns whether the given account requires follower approval - """ - if not actorJson.get('manuallyApprovesFollowers'): - return False - if actorJson['manuallyApprovesFollowers'] is True: - return True - return False - - def _createPersonBase(baseDir: str, nickname: str, domain: str, port: int, httpPrefix: str, saveToFile: bool, manualFollowerApproval: bool, diff --git a/utils.py b/utils.py index ad3e2eddf..0187f86ef 100644 --- a/utils.py +++ b/utils.py @@ -19,6 +19,16 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar +def getLockedAccount(actorJson: {}) -> bool: + """Returns whether the given account requires follower approval + """ + if not actorJson.get('manuallyApprovesFollowers'): + return False + if actorJson['manuallyApprovesFollowers'] is True: + return True + return False + + def hasUsersPath(pathStr: str) -> bool: """Whether there is a /users/ path (or equivalent) in the given string """ diff --git a/webapp_profile.py b/webapp_profile.py index 4d6e21700..eb02b0b96 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -8,6 +8,7 @@ __status__ = "Production" import os from pprint import pprint +from utils import getLockedAccount from utils import hasUsersPath from utils import getFullDomain from utils import isDormant @@ -164,6 +165,11 @@ def htmlProfileAfterSearch(cssCache: {}, displayName = searchNickname if profileJson.get('name'): displayName = profileJson['name'] + + lockedAccount = getLockedAccount(profileJson) + if lockedAccount: + displayName += '🔒' + profileDescription = '' if profileJson.get('summary'): profileDescription = profileJson['summary'] From 03044b9819cd4e57c24446132fc863e2eb0769cc Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Jan 2021 11:37:24 +0000 Subject: [PATCH 05/28] Show follows you status on person options --- follow.py | 10 +++++----- translations/ar.json | 3 ++- translations/ca.json | 3 ++- translations/cy.json | 3 ++- translations/de.json | 3 ++- translations/en.json | 3 ++- translations/es.json | 3 ++- translations/fr.json | 3 ++- translations/ga.json | 3 ++- translations/hi.json | 3 ++- translations/it.json | 3 ++- translations/ja.json | 3 ++- translations/oc.json | 3 ++- translations/pt.json | 3 ++- translations/ru.json | 3 ++- translations/zh.json | 3 ++- webapp_person_options.py | 9 +++++++++ 17 files changed, 44 insertions(+), 20 deletions(-) diff --git a/follow.py b/follow.py index ca7fb1637..25e4124ac 100644 --- a/follow.py +++ b/follow.py @@ -175,8 +175,8 @@ def followerOfPerson(baseDir: str, nickname: str, domain: str, federationList, debug, 'followers.txt') -def _isFollowerOfPerson(baseDir: str, nickname: str, domain: str, - followerNickname: str, followerDomain: str) -> bool: +def isFollowerOfPerson(baseDir: str, nickname: str, domain: str, + followerNickname: str, followerDomain: str) -> bool: """is the given nickname a follower of followerNickname? """ if ':' in domain: @@ -663,9 +663,9 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, baseDir + '/accounts/' + handleToFollow) return True - if _isFollowerOfPerson(baseDir, - nicknameToFollow, domainToFollowFull, - nickname, domainFull): + if isFollowerOfPerson(baseDir, + nicknameToFollow, domainToFollowFull, + nickname, domainFull): if debug: print('DEBUG: ' + nickname + '@' + domain + ' is already a follower of ' + diff --git a/translations/ar.json b/translations/ar.json index c0288c1e5..7cda02d25 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -349,5 +349,6 @@ "Unfilter words": "الكلمات غير المصفاة", "Show Accounts": "إظهار الحسابات", "Peertube Instances": "مثيلات Peertube", - "Show video previews for the following Peertube sites.": "إظهار معاينات الفيديو لمواقع Peertube التالية." + "Show video previews for the following Peertube sites.": "إظهار معاينات الفيديو لمواقع Peertube التالية.", + "Follows you": "يتبعك" } diff --git a/translations/ca.json b/translations/ca.json index ffc4725d9..d17bbc050 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -349,5 +349,6 @@ "Unfilter words": "Paraules sense filtre", "Show Accounts": "Mostra comptes", "Peertube Instances": "Instàncies de Peertube", - "Show video previews for the following Peertube sites.": "Mostra les previsualitzacions de vídeo dels següents llocs de Peertube." + "Show video previews for the following Peertube sites.": "Mostra les previsualitzacions de vídeo dels següents llocs de Peertube.", + "Follows you": "Et segueix" } diff --git a/translations/cy.json b/translations/cy.json index cd659af7a..8db9b525b 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -349,5 +349,6 @@ "Unfilter words": "Geiriau di-hid", "Show Accounts": "Dangos Cyfrifon", "Peertube Instances": "Camau Peertube", - "Show video previews for the following Peertube sites.": "Dangos rhagolygon fideo ar gyfer y safleoedd Peertube canlynol." + "Show video previews for the following Peertube sites.": "Dangos rhagolygon fideo ar gyfer y safleoedd Peertube canlynol.", + "Follows you": "Yn eich dilyn chi" } diff --git a/translations/de.json b/translations/de.json index e0e757b7d..a82a4aaa7 100644 --- a/translations/de.json +++ b/translations/de.json @@ -349,5 +349,6 @@ "Unfilter words": "Wörter herausfiltern", "Show Accounts": "Konten anzeigen", "Peertube Instances": "Peertube-Instanzen", - "Show video previews for the following Peertube sites.": "Zeigen Sie eine Videovorschau für die folgenden Peertube-Websites an." + "Show video previews for the following Peertube sites.": "Zeigen Sie eine Videovorschau für die folgenden Peertube-Websites an.", + "Follows you": "Folgt dir" } diff --git a/translations/en.json b/translations/en.json index 6ea42b2c6..9e93dcb82 100644 --- a/translations/en.json +++ b/translations/en.json @@ -349,5 +349,6 @@ "Unfilter words": "Unfilter words", "Show Accounts": "Show Accounts", "Peertube Instances": "Peertube Instances", - "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites." + "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites.", + "Follows you": "Follows you" } diff --git a/translations/es.json b/translations/es.json index 49293f2e5..edc36088f 100644 --- a/translations/es.json +++ b/translations/es.json @@ -349,5 +349,6 @@ "Unfilter words": "Palabras sin filtrar", "Show Accounts": "Mostrar cuentas", "Peertube Instances": "Instancias de Peertube", - "Show video previews for the following Peertube sites.": "Muestre vistas previas de video para los siguientes sitios de Peertube." + "Show video previews for the following Peertube sites.": "Muestre vistas previas de video para los siguientes sitios de Peertube.", + "Follows you": "Te sigue" } diff --git a/translations/fr.json b/translations/fr.json index ede3e55fa..838d18388 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -349,5 +349,6 @@ "Unfilter words": "Mots non filtrés", "Show Accounts": "Afficher les comptes", "Peertube Instances": "Instances Peertube", - "Show video previews for the following Peertube sites.": "Afficher des aperçus vidéo pour les sites Peertube suivants." + "Show video previews for the following Peertube sites.": "Afficher des aperçus vidéo pour les sites Peertube suivants.", + "Follows you": "Vous suit" } diff --git a/translations/ga.json b/translations/ga.json index af58fe18d..396d3ae8d 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -349,5 +349,6 @@ "Unfilter words": "Focail neamhleithleacha", "Show Accounts": "Taispeáin Cuntais", "Peertube Instances": "Imeachtaí Peertube", - "Show video previews for the following Peertube sites.": "Taispeáin réamhamharcanna físe do na suíomhanna Peertube seo a leanas." + "Show video previews for the following Peertube sites.": "Taispeáin réamhamharcanna físe do na suíomhanna Peertube seo a leanas.", + "Follows you": "Leanann tú" } diff --git a/translations/hi.json b/translations/hi.json index 9c0f78c50..05878a580 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -349,5 +349,6 @@ "Unfilter words": "अनफ़िल्टर शब्द", "Show Accounts": "खाते दिखाएं", "Peertube Instances": "Peertube उदाहरण", - "Show video previews for the following Peertube sites.": "निम्नलिखित Peertube साइटों के लिए वीडियो पूर्वावलोकन दिखाएं।" + "Show video previews for the following Peertube sites.": "निम्नलिखित Peertube साइटों के लिए वीडियो पूर्वावलोकन दिखाएं।", + "Follows you": "आपका पीछा करता है" } diff --git a/translations/it.json b/translations/it.json index 36b2fdc10..72d777ee1 100644 --- a/translations/it.json +++ b/translations/it.json @@ -349,5 +349,6 @@ "Unfilter words": "Parole non filtrate", "Show Accounts": "Mostra account", "Peertube Instances": "Istanze di Peertube", - "Show video previews for the following Peertube sites.": "Mostra le anteprime dei video per i seguenti siti Peertube." + "Show video previews for the following Peertube sites.": "Mostra le anteprime dei video per i seguenti siti Peertube.", + "Follows you": "Ti segue" } diff --git a/translations/ja.json b/translations/ja.json index 079f7bd07..6287b1ab7 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -349,5 +349,6 @@ "Unfilter words": "単語のフィルタリングを解除する", "Show Accounts": "アカウントを表示する", "Peertube Instances": "Peertubeインスタンス", - "Show video previews for the following Peertube sites.": "次のPeertubeサイトのビデオプレビューを表示します。" + "Show video previews for the following Peertube sites.": "次のPeertubeサイトのビデオプレビューを表示します。", + "Follows you": "あなたについていきます" } diff --git a/translations/oc.json b/translations/oc.json index 2b7e26f7e..eb198fb68 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -345,5 +345,6 @@ "Unfilter words": "Unfilter words", "Show Accounts": "Show Accounts", "Peertube Instances": "Peertube Instances", - "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites." + "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites.", + "Follows you": "Follows you" } diff --git a/translations/pt.json b/translations/pt.json index 67b1f3891..dfa17dc02 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -349,5 +349,6 @@ "Unfilter words": "Palavras sem filtro", "Show Accounts": "Mostrar contas", "Peertube Instances": "Instâncias Peertube", - "Show video previews for the following Peertube sites.": "Mostrar visualizações de vídeo para os seguintes sites Peertube." + "Show video previews for the following Peertube sites.": "Mostrar visualizações de vídeo para os seguintes sites Peertube.", + "Follows you": "Segue você" } diff --git a/translations/ru.json b/translations/ru.json index 3236b46fa..855c07f52 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -349,5 +349,6 @@ "Unfilter words": "Не фильтровать слова", "Show Accounts": "Показать счета", "Peertube Instances": "Экземпляры Peertube", - "Show video previews for the following Peertube sites.": "Показать превью видео для следующих сайтов Peertube." + "Show video previews for the following Peertube sites.": "Показать превью видео для следующих сайтов Peertube.", + "Follows you": "Следует за вами" } diff --git a/translations/zh.json b/translations/zh.json index fe7a8e92e..a2be25403 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -349,5 +349,6 @@ "Unfilter words": "未过滤字词", "Show Accounts": "显示帐户", "Peertube Instances": "Peertube实例", - "Show video previews for the following Peertube sites.": "显示以下Peertube网站的视频预览。" + "Show video previews for the following Peertube sites.": "显示以下Peertube网站的视频预览。", + "Follows you": "跟着你" } diff --git a/webapp_person_options.py b/webapp_person_options.py index 60ecc2eff..30d623e0e 100644 --- a/webapp_person_options.py +++ b/webapp_person_options.py @@ -18,6 +18,7 @@ from utils import removeHtml from utils import getDomainFromActor from utils import getNicknameFromActor from blocking import isBlocked +from follow import isFollowerOfPerson from follow import isFollowingActor from followingCalendar import receivingCalendarEvents from webapp_utils import htmlHeaderWithExternalStyle @@ -62,6 +63,7 @@ def htmlPersonOptions(defaultTimeline: str, blockStr = 'Block' nickname = None optionsNickname = None + followsYou = False if originPathStr.startswith('/users/'): nickname = originPathStr.split('/users/')[1] if '/' in nickname: @@ -77,6 +79,10 @@ def htmlPersonOptions(defaultTimeline: str, optionsNickname = getNicknameFromActor(optionsActor) optionsDomainFull = getFullDomain(optionsDomain, optionsPort) + followsYou = \ + isFollowerOfPerson(baseDir, + nickname, domain, + optionsNickname, optionsDomainFull) if isBlocked(baseDir, nickname, domain, optionsNickname, optionsDomainFull): blockStr = 'Block' @@ -120,6 +126,9 @@ def htmlPersonOptions(defaultTimeline: str, optionsStr += \ '

' + translate['Options for'] + \ ' @' + handleShown + '

\n' + if followsYou: + optionsStr += \ + '

' + translate['Follows you'] + '

\n' if emailAddress: optionsStr += \ '

' + translate['Email'] + \ From 0accfaea3986244bdc25edf334fb17928f23a5de Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 2 Jan 2021 12:00:07 +0000 Subject: [PATCH 06/28] Show follows you status on profile after search --- webapp_profile.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/webapp_profile.py b/webapp_profile.py index eb02b0b96..65d29a592 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -38,6 +38,7 @@ from tox import getToxAddress from briar import getBriarAddress from jami import getJamiAddress from filters import isFiltered +from follow import isFollowerOfPerson from webapp_frontscreen import htmlFrontScreen from webapp_utils import scheduledPostsExist from webapp_utils import getPersonAvatarUrl @@ -170,6 +171,12 @@ def htmlProfileAfterSearch(cssCache: {}, if lockedAccount: displayName += '🔒' + followsYou = \ + isFollowerOfPerson(baseDir, + nickname, domain, + searchNickname, + searchDomainFull) + profileDescription = '' if profileJson.get('summary'): profileDescription = profileJson['summary'] @@ -224,7 +231,7 @@ def htmlProfileAfterSearch(cssCache: {}, searchNickname, searchDomainFull, translate, - displayName, + displayName, followsYou, profileDescriptionShort, avatarUrl, imageUrl) @@ -330,6 +337,7 @@ def _getProfileHeaderAfterSearch(baseDir: str, searchDomainFull: str, translate: {}, displayName: str, + followsYou: bool, profileDescriptionShort: str, avatarUrl: str, imageUrl: str) -> str: """The header of a searched for handle, containing background @@ -352,6 +360,8 @@ def _getProfileHeaderAfterSearch(baseDir: str, htmlStr += '

' + displayName + '

\n' htmlStr += \ '

@' + searchNickname + '@' + searchDomainFull + '
\n' + if followsYou: + htmlStr += '

' + translate['Follows you'] + '

\n' htmlStr += '

' + profileDescriptionShort + '

\n' htmlStr += ' \n' htmlStr += ' \n\n' From c09596f1e2e2d39a86124e1f5dfe96a91eebe29d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 09:44:33 +0000 Subject: [PATCH 07/28] More verbose --- inbox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inbox.py b/inbox.py index 1c70ee491..2cc8037eb 100644 --- a/inbox.py +++ b/inbox.py @@ -2679,9 +2679,9 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queue.pop(0) continue - # check the signature + # check the http header signature if debug: - print('DEBUG: checking http headers') + print('DEBUG: checking http header signature') pprint(queueJson['httpHeaders']) postStr = json.dumps(queueJson['post']) if not verifyPostHeaders(httpPrefix, @@ -2700,7 +2700,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, continue if debug: - print('DEBUG: Signature check success') + print('DEBUG: http header signature check success') # set the id to the same as the post filename # This makes the filename and the id consistent From e8290d99d5da9e7ed57d875b640f03aee1587eca Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 14:25:20 +0000 Subject: [PATCH 08/28] Check that json signatured are present on inbox posts --- inbox.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/inbox.py b/inbox.py index 2cc8037eb..1c40f7411 100644 --- a/inbox.py +++ b/inbox.py @@ -10,6 +10,7 @@ import json import os import datetime import time +from jsonldsig import jsonldVerify from utils import hasUsersPath from utils import validPostDate from utils import getFullDomain @@ -2702,6 +2703,38 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if debug: print('DEBUG: http header signature check success') + hasJsonSig = True + + if not queueJson['post'].get('signature'): + print('WARN: jsonld inbox signature signature missing') + hasJsonSig = False + + if not queueJson['post']['signature'].get('type'): + print('WARN: jsonld inbox signature type missing') + hasJsonSig = False + +# if not jsonldVerify(queueJson['post'], pubKey): +# hasJsonSig = False +# if debug: +# print('**************************************') +# print('WARN: jsonld signature check failed ' + +# str(queueJson['post'])) +# print('--------------------------------------') +# print(keyId) +# print(pubKey) +# print('**************************************') +# else: +# if debug: +# print('jsonld inbox signature check success') +# + if not hasJsonSig: + # json signature check failed + if os.path.isfile(queueFilename): + os.remove(queueFilename) + if len(queue) > 0: + queue.pop(0) + continue + # set the id to the same as the post filename # This makes the filename and the id consistent # if queueJson['post'].get('id'): From 1a74ec6d53cd5f26a28de0975d131bcbe2ed4414 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 14:31:35 +0000 Subject: [PATCH 09/28] Failing test for json signature failure --- tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests.py b/tests.py index b9bac2492..6c86c225e 100644 --- a/tests.py +++ b/tests.py @@ -2030,6 +2030,9 @@ def testJsonld(): assert(len(signedDocument['signature']['signatureValue']) > 50) assert(signedDocument['signature']['type'] == 'RsaSignatureSuite2017') assert(jsonldVerify(signedDocument, publicKeyPem)) + # alter the signed document + # signedDocument['object']['content'] = 'forged content' + # assert(not jsonldVerify(signedDocument, publicKeyPem)) def testSiteIsActive(): From 8c3ca5a69c3e4a9d59b345190a789f03c492aad0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 14:34:27 +0000 Subject: [PATCH 10/28] Valid --- tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 6c86c225e..740f092bd 100644 --- a/tests.py +++ b/tests.py @@ -1981,7 +1981,7 @@ def testJsonld(): "description": "My json document", "numberField": 83582, "object": { - "content": "Some content" + "content": "valid content" } } # privateKeyPem, publicKeyPem = generateRSAKey() From 183141ee803ecc06c660ca4436d546aad135ff7d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 14:36:16 +0000 Subject: [PATCH 11/28] Show from key --- inbox.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/inbox.py b/inbox.py index 1c40f7411..297d4ab82 100644 --- a/inbox.py +++ b/inbox.py @@ -2706,11 +2706,13 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, hasJsonSig = True if not queueJson['post'].get('signature'): - print('WARN: jsonld inbox signature signature missing') + print('WARN: jsonld inbox signature signature missing from ' + + keyId) hasJsonSig = False if not queueJson['post']['signature'].get('type'): - print('WARN: jsonld inbox signature type missing') + print('WARN: jsonld inbox signature type missing from ' + + keyId) hasJsonSig = False # if not jsonldVerify(queueJson['post'], pubKey): From be14587011147b539b038cbaa49d14a8793f44ac Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 14:44:57 +0000 Subject: [PATCH 12/28] Only warn about json signature failures --- inbox.py | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/inbox.py b/inbox.py index 297d4ab82..d0c357339 100644 --- a/inbox.py +++ b/inbox.py @@ -2703,39 +2703,24 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if debug: print('DEBUG: http header signature check success') - hasJsonSig = True - if not queueJson['post'].get('signature'): print('WARN: jsonld inbox signature signature missing from ' + keyId) - hasJsonSig = False - - if not queueJson['post']['signature'].get('type'): - print('WARN: jsonld inbox signature type missing from ' + - keyId) - hasJsonSig = False - -# if not jsonldVerify(queueJson['post'], pubKey): -# hasJsonSig = False -# if debug: -# print('**************************************') -# print('WARN: jsonld signature check failed ' + -# str(queueJson['post'])) -# print('--------------------------------------') -# print(keyId) -# print(pubKey) -# print('**************************************') # else: -# if debug: -# print('jsonld inbox signature check success') +# if not jsonldVerify(queueJson['post'], pubKey): +# hasJsonSig = False +# if debug: +# print('**************************************') +# print('WARN: jsonld signature check failed ' + +# str(queueJson['post'])) +# print('--------------------------------------') +# print(keyId) +# print(pubKey) +# print('**************************************') +# else: +# if debug: +# print('jsonld inbox signature check success') # - if not hasJsonSig: - # json signature check failed - if os.path.isfile(queueFilename): - os.remove(queueFilename) - if len(queue) > 0: - queue.pop(0) - continue # set the id to the same as the post filename # This makes the filename and the id consistent From b9d33296a135db03a530ab807ddd4a2ec26afe28 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 15:27:59 +0000 Subject: [PATCH 13/28] Check for changing json signature --- tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests.py b/tests.py index 740f092bd..70421cd3c 100644 --- a/tests.py +++ b/tests.py @@ -2034,6 +2034,24 @@ def testJsonld(): # signedDocument['object']['content'] = 'forged content' # assert(not jsonldVerify(signedDocument, publicKeyPem)) + jldDocument2 = { + "description": "Another json document", + "numberField": 13353, + "object": { + "content": "More content" + } + } + signedDocument2 = testSignJsonld(jldDocument2, privateKeyPem) + assert(signedDocument2) + assert(signedDocument2.get('signature')) + assert(signedDocument2['signature'].get('signatureValue')) + # changed signature on different document + if signedDocument['signature']['signatureValue'] == \ + signedDocument2['signature']['signatureValue']: + print('json signature has not changed for different documents') +# assert(signedDocument['signature']['signatureValue'] != +# signedDocument2['signature']['signatureValue']) + def testSiteIsActive(): print('testSiteIsActive') From 77f965162cf750e4350228ed0446bdcc8c443caa Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:20:25 +0000 Subject: [PATCH 14/28] Enable checking of json signatures on inbox posts --- daemon.py | 4 +++- inbox.py | 44 ++++++++++++++++++++++++++------------------ jsonldsig.py | 8 ++++++++ pyjsonld.py | 9 ++++++++- tests.py | 10 ++++++---- 5 files changed, 51 insertions(+), 24 deletions(-) diff --git a/daemon.py b/daemon.py index 2d1ec1721..ca8ed0b1f 100644 --- a/daemon.py +++ b/daemon.py @@ -1079,6 +1079,8 @@ class PubServer(BaseHTTPRequestHandler): elif self.headers.get('content-length'): headersDict['content-length'] = self.headers['content-length'] + originalMessageJson = messageJson.copy() + # For follow activities add a 'to' field, which is a copy # of the object field messageJson, toFieldExists = \ @@ -1097,7 +1099,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, nickname, self.server.domainFull, - messageJson, + messageJson, originalMessageJson, messageBytesDecoded, headersDict, self.path, diff --git a/inbox.py b/inbox.py index d0c357339..37a476dcc 100644 --- a/inbox.py +++ b/inbox.py @@ -313,6 +313,7 @@ def inboxPermittedMessage(domain: str, messageJson: {}, def savePostToInboxQueue(baseDir: str, httpPrefix: str, nickname: str, domain: str, postJsonObject: {}, + originalPostJsonObject: {}, messageBytes: str, httpHeaders: {}, postPath: str, debug: bool) -> str: @@ -437,6 +438,7 @@ def savePostToInboxQueue(baseDir: str, httpPrefix: str, 'httpHeaders': httpHeaders, 'path': postPath, 'post': postJsonObject, + 'original': originalPostJsonObject, 'digest': digest, 'filename': filename, 'destination': destination @@ -2703,24 +2705,30 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if debug: print('DEBUG: http header signature check success') - if not queueJson['post'].get('signature'): - print('WARN: jsonld inbox signature signature missing from ' + - keyId) -# else: -# if not jsonldVerify(queueJson['post'], pubKey): -# hasJsonSig = False -# if debug: -# print('**************************************') -# print('WARN: jsonld signature check failed ' + -# str(queueJson['post'])) -# print('--------------------------------------') -# print(keyId) -# print(pubKey) -# print('**************************************') -# else: -# if debug: -# print('jsonld inbox signature check success') -# + # should the json signature be checked? + checkJsonSignature = False + if queueJson['original'].get('@context'): + checkJsonSignature = True + if not queueJson['original'].get('signature'): + print('WARN: jsonld inbox signature signature missing from ' + + keyId) + checkJsonSignature = False + + # check json signature + if checkJsonSignature: + # use the original json message received, not one which may have + # been modified along the way + if not jsonldVerify(queueJson['original'], pubKey): + print('WARN: jsonld signature check failed ' + + keyId + ' ' + pubKey + ' ' + + str(queueJson['original'])) + if os.path.isfile(queueFilename): + os.remove(queueFilename) + if len(queue) > 0: + queue.pop(0) + continue + else: + print('jsonld inbox signature check success') # set the id to the same as the post filename # This makes the filename and the id consistent diff --git a/jsonldsig.py b/jsonldsig.py index 30a09e961..df7e93a57 100644 --- a/jsonldsig.py +++ b/jsonldsig.py @@ -116,6 +116,9 @@ def jsonldSign(jldDocument: {}, privateKeyPem: str) -> {}: """ Produces a signed JSON-LD document with a Json Web Signature """ + if not jldDocument.get('@context'): + print('WARN: json document must have @context to sign') + return jldDocument jldDocument = deepcopy(jldDocument) normalizedJldHash = _jsonldNormalize(jldDocument) jwsSignature = _signJws(normalizedJldHash, privateKeyPem) @@ -135,6 +138,11 @@ def jsonldVerify(signedJldDocument: {}, publicKeyPem: str) -> bool: """ Verifies the Json Web Signature of a signed JSON-LD Document """ + if not isinstance(signedJldDocument, dict): + return False + if not signedJldDocument.get('@context'): + print('json document must have @context') + return False signedJldDocument = deepcopy(signedJldDocument) signature = signedJldDocument.pop('signature') jwsSignature = signature['signatureValue'].encode('utf-8') diff --git a/pyjsonld.py b/pyjsonld.py index f0f55b41b..5f31ca0e5 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -234,7 +234,7 @@ def link(input_, ctx, options=None): return frame(input, frame, options) -def normalize(input_, options=None): +def normalize(input_: {}, options=None): """ Performs JSON-LD normalization. @@ -1016,6 +1016,13 @@ class JsonLdProcessor(object): 'Could not convert input to RDF dataset before normalization.', 'jsonld.NormalizeError', cause=cause) + # check that the data is not empty + if '@default' in dataset: + if not dataset['@default']: + raise JsonLdError( + 'Could not convert input to RDF dataset.', + 'jsonld.NormalizeError', cause=None) + # do normalization return self._normalize(dataset, options) diff --git a/tests.py b/tests.py index 70421cd3c..bc9003482 100644 --- a/tests.py +++ b/tests.py @@ -1978,6 +1978,7 @@ def testRemoveTextFormatting(): def testJsonld(): print("testJsonld") jldDocument = { + "@context": "https://www.w3.org/ns/activitystreams", "description": "My json document", "numberField": 83582, "object": { @@ -2031,10 +2032,11 @@ def testJsonld(): assert(signedDocument['signature']['type'] == 'RsaSignatureSuite2017') assert(jsonldVerify(signedDocument, publicKeyPem)) # alter the signed document - # signedDocument['object']['content'] = 'forged content' - # assert(not jsonldVerify(signedDocument, publicKeyPem)) + signedDocument['object']['content'] = 'forged content' + assert(not jsonldVerify(signedDocument, publicKeyPem)) jldDocument2 = { + "@context": "https://www.w3.org/ns/activitystreams", "description": "Another json document", "numberField": 13353, "object": { @@ -2049,8 +2051,8 @@ def testJsonld(): if signedDocument['signature']['signatureValue'] == \ signedDocument2['signature']['signatureValue']: print('json signature has not changed for different documents') -# assert(signedDocument['signature']['signatureValue'] != -# signedDocument2['signature']['signatureValue']) + assert(signedDocument['signature']['signatureValue'] != + signedDocument2['signature']['signatureValue']) def testSiteIsActive(): From b0eaa6835d18519d461e23a116a9745a5e9e19b1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:25:23 +0000 Subject: [PATCH 15/28] Consistent message --- inbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inbox.py b/inbox.py index 37a476dcc..3ecd8a6f5 100644 --- a/inbox.py +++ b/inbox.py @@ -2719,7 +2719,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, # use the original json message received, not one which may have # been modified along the way if not jsonldVerify(queueJson['original'], pubKey): - print('WARN: jsonld signature check failed ' + + print('WARN: jsonld inbox signature check failed ' + keyId + ' ' + pubKey + ' ' + str(queueJson['original'])) if os.path.isfile(queueFilename): From adebd3c3bd73200fccc6620ce61294e61a74bee8 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:29:12 +0000 Subject: [PATCH 16/28] Remove check --- pyjsonld.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pyjsonld.py b/pyjsonld.py index 5f31ca0e5..bb3213624 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -1016,13 +1016,6 @@ class JsonLdProcessor(object): 'Could not convert input to RDF dataset before normalization.', 'jsonld.NormalizeError', cause=cause) - # check that the data is not empty - if '@default' in dataset: - if not dataset['@default']: - raise JsonLdError( - 'Could not convert input to RDF dataset.', - 'jsonld.NormalizeError', cause=None) - # do normalization return self._normalize(dataset, options) From 2449b57005d0a76c9e666986454763dad3b92921 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:34:35 +0000 Subject: [PATCH 17/28] Simplify --- inbox.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/inbox.py b/inbox.py index 3ecd8a6f5..7159b8b44 100644 --- a/inbox.py +++ b/inbox.py @@ -2705,17 +2705,9 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if debug: print('DEBUG: http header signature check success') - # should the json signature be checked? - checkJsonSignature = False - if queueJson['original'].get('@context'): - checkJsonSignature = True - if not queueJson['original'].get('signature'): - print('WARN: jsonld inbox signature signature missing from ' + - keyId) - checkJsonSignature = False - # check json signature - if checkJsonSignature: + if queueJson['original'].get('@context') and \ + queueJson['original'].get('signature'): # use the original json message received, not one which may have # been modified along the way if not jsonldVerify(queueJson['original'], pubKey): From bc575dc6c0ff22ab80b6b631f67d1c165f4387e0 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:38:23 +0000 Subject: [PATCH 18/28] Debug --- jsonldsig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jsonldsig.py b/jsonldsig.py index df7e93a57..a9e14ecb3 100644 --- a/jsonldsig.py +++ b/jsonldsig.py @@ -93,6 +93,7 @@ def _verifyJws(payload: {}, jwsSignature: str, publicKeyPem: str) -> bool: """ Verifies a signature using the given public key """ + print('inbox signature: ' + str(jwsSignature)) encodedHeader, encodedSignature = jwsSignature.split(b'..') signature = _b64safeDecode(encodedSignature) payload = b'.'.join([encodedHeader, payload]) From edf0c8880e8d9b52a5ab8e44e01a7207cfab82ba Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:40:26 +0000 Subject: [PATCH 19/28] Debug --- inbox.py | 1 + jsonldsig.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/inbox.py b/inbox.py index 7159b8b44..5c2f0e2f4 100644 --- a/inbox.py +++ b/inbox.py @@ -2710,6 +2710,7 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queueJson['original'].get('signature'): # use the original json message received, not one which may have # been modified along the way + print('inbox signature: ' + str(queueJson['original'])) if not jsonldVerify(queueJson['original'], pubKey): print('WARN: jsonld inbox signature check failed ' + keyId + ' ' + pubKey + ' ' + diff --git a/jsonldsig.py b/jsonldsig.py index a9e14ecb3..df7e93a57 100644 --- a/jsonldsig.py +++ b/jsonldsig.py @@ -93,7 +93,6 @@ def _verifyJws(payload: {}, jwsSignature: str, publicKeyPem: str) -> bool: """ Verifies a signature using the given public key """ - print('inbox signature: ' + str(jwsSignature)) encodedHeader, encodedSignature = jwsSignature.split(b'..') signature = _b64safeDecode(encodedSignature) payload = b'.'.join([encodedHeader, payload]) From 5a327d281efb17bbb1184bc2fcf3cb0141cfd006 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 18:42:39 +0000 Subject: [PATCH 20/28] Debug --- inbox.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/inbox.py b/inbox.py index 5c2f0e2f4..db29ffb91 100644 --- a/inbox.py +++ b/inbox.py @@ -2710,18 +2710,19 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queueJson['original'].get('signature'): # use the original json message received, not one which may have # been modified along the way + print('inbox signature: ' + str(pubKey)) print('inbox signature: ' + str(queueJson['original'])) - if not jsonldVerify(queueJson['original'], pubKey): - print('WARN: jsonld inbox signature check failed ' + - keyId + ' ' + pubKey + ' ' + - str(queueJson['original'])) - if os.path.isfile(queueFilename): - os.remove(queueFilename) - if len(queue) > 0: - queue.pop(0) - continue - else: - print('jsonld inbox signature check success') +# if not jsonldVerify(queueJson['original'], pubKey): +# print('WARN: jsonld inbox signature check failed ' + +# keyId + ' ' + pubKey + ' ' + +# str(queueJson['original'])) +# if os.path.isfile(queueFilename): +# os.remove(queueFilename) +# if len(queue) > 0: +# queue.pop(0) +# continue +# else: +# print('jsonld inbox signature check success') # set the id to the same as the post filename # This makes the filename and the id consistent From a8906b25d7e019d220ac8204b9e830372c8d4799 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 19:08:39 +0000 Subject: [PATCH 21/28] Check for signature without header --- inbox.py | 24 +++++++++++------------- jsonldsig.py | 10 +++++++--- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/inbox.py b/inbox.py index db29ffb91..7159b8b44 100644 --- a/inbox.py +++ b/inbox.py @@ -2710,19 +2710,17 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, queueJson['original'].get('signature'): # use the original json message received, not one which may have # been modified along the way - print('inbox signature: ' + str(pubKey)) - print('inbox signature: ' + str(queueJson['original'])) -# if not jsonldVerify(queueJson['original'], pubKey): -# print('WARN: jsonld inbox signature check failed ' + -# keyId + ' ' + pubKey + ' ' + -# str(queueJson['original'])) -# if os.path.isfile(queueFilename): -# os.remove(queueFilename) -# if len(queue) > 0: -# queue.pop(0) -# continue -# else: -# print('jsonld inbox signature check success') + if not jsonldVerify(queueJson['original'], pubKey): + print('WARN: jsonld inbox signature check failed ' + + keyId + ' ' + pubKey + ' ' + + str(queueJson['original'])) + if os.path.isfile(queueFilename): + os.remove(queueFilename) + if len(queue) > 0: + queue.pop(0) + continue + else: + print('jsonld inbox signature check success') # set the id to the same as the post filename # This makes the filename and the id consistent diff --git a/jsonldsig.py b/jsonldsig.py index df7e93a57..87e416220 100644 --- a/jsonldsig.py +++ b/jsonldsig.py @@ -93,9 +93,13 @@ def _verifyJws(payload: {}, jwsSignature: str, publicKeyPem: str) -> bool: """ Verifies a signature using the given public key """ - encodedHeader, encodedSignature = jwsSignature.split(b'..') - signature = _b64safeDecode(encodedSignature) - payload = b'.'.join([encodedHeader, payload]) + if b'..' in jwsSignature: + encodedHeader, encodedSignature = jwsSignature.split(b'..') + signature = _b64safeDecode(encodedSignature) + payload = b'.'.join([encodedHeader, payload]) + else: + signature = _b64safeDecode(jwsSignature) + payload = b'.'.join([payload]) return _verifyRs256(payload, signature, publicKeyPem) From 5fc36f6ebb10448bd2b0331a1b62382d5da4f292 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 19:11:59 +0000 Subject: [PATCH 22/28] Debug --- inbox.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/inbox.py b/inbox.py index 7159b8b44..7b1eaac0a 100644 --- a/inbox.py +++ b/inbox.py @@ -2714,11 +2714,11 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, print('WARN: jsonld inbox signature check failed ' + keyId + ' ' + pubKey + ' ' + str(queueJson['original'])) - if os.path.isfile(queueFilename): - os.remove(queueFilename) - if len(queue) > 0: - queue.pop(0) - continue +# if os.path.isfile(queueFilename): +# os.remove(queueFilename) +# if len(queue) > 0: +# queue.pop(0) +# continue else: print('jsonld inbox signature check success') From 6aa1cc8389f819a01d8c107e1b1ca1dbc6913fc1 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sun, 3 Jan 2021 19:20:28 +0000 Subject: [PATCH 23/28] Extra json signature checks --- inbox.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inbox.py b/inbox.py index 7b1eaac0a..59987d7c3 100644 --- a/inbox.py +++ b/inbox.py @@ -2706,8 +2706,16 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, print('DEBUG: http header signature check success') # check json signature + checkJsonSignature = False if queueJson['original'].get('@context') and \ queueJson['original'].get('signature'): + if isinstance(queueJson['original']['signature'], dict): + if queueJson['original']['signature'].get('type') and \ + queueJson['original']['signature'].get('signatureValue'): + if queueJson['original']['signature']['type'] == \ + 'RsaSignature2017': + checkJsonSignature = True + if checkJsonSignature: # use the original json message received, not one which may have # been modified along the way if not jsonldVerify(queueJson['original'], pubKey): From 9bdfec94f0936a08189f6e11d5db6cfb5906f120 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 4 Jan 2021 19:02:24 +0000 Subject: [PATCH 24/28] Fix jsonld signatures Also some schemas are no longer remotely accessed --- daemon.py | 6 + inbox.py | 25 +-- jsonldsig.py | 167 ---------------- linked_data_sig.py | 84 ++++++++ posts.py | 8 +- pyjsonld.py | 486 +++++++++++++++++++++++++++++++++++++++++++++ tests.py | 23 ++- 7 files changed, 610 insertions(+), 189 deletions(-) delete mode 100644 jsonldsig.py create mode 100644 linked_data_sig.py diff --git a/daemon.py b/daemon.py index ca8ed0b1f..4c93a9ac4 100644 --- a/daemon.py +++ b/daemon.py @@ -13235,6 +13235,12 @@ class PubServer(BaseHTTPRequestHandler): if self.server.debug: print('DEBUG: Check message has params') + if not messageJson: + self.send_response(403) + self.end_headers() + self.server.POSTbusy = False + return + if self.path.endswith('/inbox') or \ self.path == '/sharedInbox': if not inboxMessageHasParams(messageJson): diff --git a/inbox.py b/inbox.py index 59987d7c3..6ad5daecf 100644 --- a/inbox.py +++ b/inbox.py @@ -10,7 +10,7 @@ import json import os import datetime import time -from jsonldsig import jsonldVerify +from linked_data_sig import verifyJsonSignature from utils import hasUsersPath from utils import validPostDate from utils import getFullDomain @@ -2705,28 +2705,29 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, if debug: print('DEBUG: http header signature check success') - # check json signature + # check if a json signature exists on this post checkJsonSignature = False if queueJson['original'].get('@context') and \ queueJson['original'].get('signature'): if isinstance(queueJson['original']['signature'], dict): - if queueJson['original']['signature'].get('type') and \ - queueJson['original']['signature'].get('signatureValue'): - if queueJson['original']['signature']['type'] == \ - 'RsaSignature2017': + # see https://tools.ietf.org/html/rfc7515 + jwebsig = queueJson['original']['signature'] + # signature exists and is of the expected type + if jwebsig.get('type') and jwebsig.get('signatureValue'): + if jwebsig['type'] == 'RsaSignature2017': checkJsonSignature = True if checkJsonSignature: # use the original json message received, not one which may have # been modified along the way - if not jsonldVerify(queueJson['original'], pubKey): + if not verifyJsonSignature(queueJson['original'], pubKey): print('WARN: jsonld inbox signature check failed ' + keyId + ' ' + pubKey + ' ' + str(queueJson['original'])) -# if os.path.isfile(queueFilename): -# os.remove(queueFilename) -# if len(queue) > 0: -# queue.pop(0) -# continue + if os.path.isfile(queueFilename): + os.remove(queueFilename) + if len(queue) > 0: + queue.pop(0) + continue else: print('jsonld inbox signature check success') diff --git a/jsonldsig.py b/jsonldsig.py deleted file mode 100644 index 87e416220..000000000 --- a/jsonldsig.py +++ /dev/null @@ -1,167 +0,0 @@ -__filename__ = "jsonldsig.py" -__author__ = "Bob Mottram" -__credits__ = ['Based on ' + - 'https://github.com/WebOfTrustInfo/ld-signatures-python'] -__license__ = "AGPL3+" -__version__ = "1.1.0" -__maintainer__ = "Bob Mottram" -__email__ = "bob@freedombone.net" -__status__ = "Production" - -from copy import deepcopy -from datetime import datetime - -import pytz - -try: - from Cryptodome.PublicKey import RSA - from Cryptodome.Hash import SHA256 - from Cryptodome.Signature import pkcs1_5 as PKCS1_v1_5 -except ImportError: - from Crypto.PublicKey import RSA - from Crypto.Hash import SHA256 - from Crypto.Signature import PKCS1_v1_5 - -from pyjsonld import normalize - -import base64 -import json - - -def _b64safeEncode(payload: {}) -> str: - """ - b64 url safe encoding with the padding removed. - """ - return base64.urlsafe_b64encode(payload).rstrip(b'=') - - -def _b64safeDecode(payload: {}) -> str: - """ - b64 url safe decoding with the padding added. - """ - return base64.urlsafe_b64decode(payload + b'=' * (4 - len(payload) % 4)) - - -def _normalizeJson(payload: {}) -> str: - """ - Normalize with URDNA2015 - """ - return json.dumps(payload, separators=(',', ':'), - sort_keys=True).encode('utf-8') - - -def _signRs256(payload: {}, privateKeyPem: str) -> str: - """ - Produce a RS256 signature of the payload - """ - key = RSA.importKey(privateKeyPem) - signer = PKCS1_v1_5.new(key) - signature = signer.sign(SHA256.new(payload)) - return signature - - -def _verifyRs256(payload: {}, signature: str, publicKeyPem: str) -> bool: - """ - Verifies a RS256 signature - """ - key = RSA.importKey(publicKeyPem) - verifier = PKCS1_v1_5.new(key) - return verifier.verify(SHA256.new(payload), signature) - - -def _signJws(payload: {}, privateKeyPem: str) -> str: - """ - Prepare payload to sign - """ - header = { - 'alg': 'RS256', - 'b64': False, - 'crit': ['b64'] - } - normalizedJson = _normalizeJson(header) - encodedHeader = _b64safeEncode(normalizedJson) - preparedPayload = b'.'.join([encodedHeader, payload]) - - signature = _signRs256(preparedPayload, privateKeyPem) - encodedSignature = _b64safeEncode(signature) - jwsSignature = b'..'.join([encodedHeader, encodedSignature]) - - return jwsSignature - - -def _verifyJws(payload: {}, jwsSignature: str, publicKeyPem: str) -> bool: - """ - Verifies a signature using the given public key - """ - if b'..' in jwsSignature: - encodedHeader, encodedSignature = jwsSignature.split(b'..') - signature = _b64safeDecode(encodedSignature) - payload = b'.'.join([encodedHeader, payload]) - else: - signature = _b64safeDecode(jwsSignature) - payload = b'.'.join([payload]) - return _verifyRs256(payload, signature, publicKeyPem) - - -def _jsonldNormalize(jldDocument: str): - """ - Normalize and hash the json-ld document - """ - options = { - 'algorithm': 'URDNA2015', - 'format': 'application/nquads' - } - normalized = normalize(jldDocument, options=options) - normalizedHash = SHA256.new(data=normalized.encode('utf-8')).digest() - return normalizedHash - - -def jsonldSign(jldDocument: {}, privateKeyPem: str) -> {}: - """ - Produces a signed JSON-LD document with a Json Web Signature - """ - if not jldDocument.get('@context'): - print('WARN: json document must have @context to sign') - return jldDocument - jldDocument = deepcopy(jldDocument) - normalizedJldHash = _jsonldNormalize(jldDocument) - jwsSignature = _signJws(normalizedJldHash, privateKeyPem) - - # construct the signature document and add it to jsonld - signature = { - 'type': 'RsaSignatureSuite2017', - 'created': datetime.now(tz=pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), - 'signatureValue': jwsSignature.decode('utf-8') - } - jldDocument.update({'signature': signature}) - - return jldDocument - - -def jsonldVerify(signedJldDocument: {}, publicKeyPem: str) -> bool: - """ - Verifies the Json Web Signature of a signed JSON-LD Document - """ - if not isinstance(signedJldDocument, dict): - return False - if not signedJldDocument.get('@context'): - print('json document must have @context') - return False - signedJldDocument = deepcopy(signedJldDocument) - signature = signedJldDocument.pop('signature') - jwsSignature = signature['signatureValue'].encode('utf-8') - normalizedJldHash = _jsonldNormalize(signedJldDocument) - - return _verifyJws(normalizedJldHash, jwsSignature, publicKeyPem) - - -def testSignJsonld(jldDocument: {}, privateKeyPem: str) -> {}: - """ - Creates a test signature - """ - signedJldDocument = jsonldSign(jldDocument, privateKeyPem) - - # pop the created time key since its dynamic - signedJldDocument['signature'].pop('created') - - return signedJldDocument diff --git a/linked_data_sig.py b/linked_data_sig.py new file mode 100644 index 000000000..f36cc288f --- /dev/null +++ b/linked_data_sig.py @@ -0,0 +1,84 @@ +__filename__ = "linked_data_sig.py" +__author__ = "Bob Mottram" +__credits__ = ['Based on ' + + 'https://github.com/tsileo/little-boxes'] +__license__ = "AGPL3+" +__version__ = "1.1.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import base64 +import hashlib +from datetime import datetime + +try: + from Cryptodome.PublicKey import RSA + from Cryptodome.Hash import SHA256 + from Cryptodome.Signature import pkcs1_5 as PKCS1_v1_5 +except ImportError: + from Crypto.PublicKey import RSA + from Crypto.Hash import SHA256 + from Crypto.Signature import PKCS1_v1_5 + +from pyjsonld import normalize + + +def _options_hash(doc): + doc = dict(doc["signature"]) + for k in ["type", "id", "signatureValue"]: + if k in doc: + del doc[k] + doc["@context"] = "https://w3id.org/identity/v1" + options = { + "algorithm": "URDNA2015", + "format": "application/nquads" + } + normalized = normalize(doc, options) + h = hashlib.new("sha256") + h.update(normalized.encode("utf-8")) + return h.hexdigest() + + +def _doc_hash(doc): + doc = dict(doc) + if "signature" in doc: + del doc["signature"] + options = { + "algorithm": "URDNA2015", + "format": "application/nquads" + } + normalized = normalize(doc, options) + h = hashlib.new("sha256") + h.update(normalized.encode("utf-8")) + return h.hexdigest() + + +def verifyJsonSignature(doc: {}, publicKeyPem: str): + key = RSA.importKey(publicKeyPem) + to_be_signed = _options_hash(doc) + _doc_hash(doc) + signature = doc["signature"]["signatureValue"] + signer = PKCS1_v1_5.new(key) # type: ignore + digest = SHA256.new() + digest.update(to_be_signed.encode("utf-8")) + return signer.verify(digest, base64.b64decode(signature)) # type: ignore + + +def generateJsonSignature(doc: {}, privateKeyPem: str): + if not doc.get('actor'): + return + + options = { + "type": "RsaSignature2017", + "creator": doc["actor"] + "#main-key", + "created": datetime.utcnow().replace(microsecond=0).isoformat() + "Z", + } + doc["signature"] = options + to_be_signed = _options_hash(doc) + _doc_hash(doc) + + key = RSA.importKey(privateKeyPem) + signer = PKCS1_v1_5.new(key) + digest = SHA256.new() + digest.update(to_be_signed.encode("utf-8")) + sig = base64.b64encode(signer.sign(digest)) # type: ignore + options["signatureValue"] = sig.decode("utf-8") diff --git a/posts.py b/posts.py index c98669627..df87b2221 100644 --- a/posts.py +++ b/posts.py @@ -65,7 +65,7 @@ from blocking import isBlocked from blocking import isBlockedDomain from filters import isFiltered from git import convertPostToPatch -from jsonldsig import jsonldSign +from linked_data_sig import generateJsonSignature from petnames import resolvePetnames @@ -1794,7 +1794,8 @@ def sendPost(projectVersion: str, if not postJsonObject.get('signature'): try: - signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem) + signedPostJsonObject = postJsonObject.copy() + generateJsonSignature(signedPostJsonObject, privateKeyPem) postJsonObject = signedPostJsonObject except Exception as e: print('WARN: failed to JSON-LD sign post, ' + str(e)) @@ -2122,7 +2123,8 @@ def sendSignedJson(postJsonObject: {}, session, baseDir: str, if not postJsonObject.get('signature'): try: - signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem) + signedPostJsonObject = postJsonObject.copy() + generateJsonSignature(signedPostJsonObject, privateKeyPem) postJsonObject = signedPostJsonObject except Exception as e: print('WARN: failed to JSON-LD sign post, ' + str(e)) diff --git a/pyjsonld.py b/pyjsonld.py index bb3213624..5b178a56f 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -355,6 +355,476 @@ def parse_link_header(header): return rval +def getV1Schema() -> {}: + # https://w3id.org/identity/v1 + return { + "@context": { + "id": "@id", + "type": "@type", + "cred": "https://w3id.org/credentials#", + "dc": "http://purl.org/dc/terms/", + "identity": "https://w3id.org/identity#", + "perm": "https://w3id.org/permissions#", + "ps": "https://w3id.org/payswarm#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "sec": "https://w3id.org/security#", + "schema": "http://schema.org/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "Group": "https://www.w3.org/ns/activitystreams#Group", + "claim": {"@id": "cred:claim", "@type": "@id"}, + "credential": {"@id": "cred:credential", "@type": "@id"}, + "issued": {"@id": "cred:issued", "@type": "xsd:dateTime"}, + "issuer": {"@id": "cred:issuer", "@type": "@id"}, + "recipient": {"@id": "cred:recipient", "@type": "@id"}, + "Credential": "cred:Credential", + "CryptographicKeyCredential": "cred:CryptographicKeyCredential", + "about": {"@id": "schema:about", "@type": "@id"}, + "address": {"@id": "schema:address", "@type": "@id"}, + "addressCountry": "schema:addressCountry", + "addressLocality": "schema:addressLocality", + "addressRegion": "schema:addressRegion", + "comment": "rdfs:comment", + "created": {"@id": "dc:created", "@type": "xsd:dateTime"}, + "creator": {"@id": "dc:creator", "@type": "@id"}, + "description": "schema:description", + "email": "schema:email", + "familyName": "schema:familyName", + "givenName": "schema:givenName", + "image": {"@id": "schema:image", "@type": "@id"}, + "label": "rdfs:label", + "name": "schema:name", + "postalCode": "schema:postalCode", + "streetAddress": "schema:streetAddress", + "title": "dc:title", + "url": {"@id": "schema:url", "@type": "@id"}, + "Person": "schema:Person", + "PostalAddress": "schema:PostalAddress", + "Organization": "schema:Organization", + "identityService": { + "@id": "identity:identityService", "@type": "@id" + }, + "idp": {"@id": "identity:idp", "@type": "@id"}, + "Identity": "identity:Identity", + "paymentProcessor": "ps:processor", + "preferences": {"@id": "ps:preferences", "@type": "@vocab"}, + "cipherAlgorithm": "sec:cipherAlgorithm", + "cipherData": "sec:cipherData", + "cipherKey": "sec:cipherKey", + "digestAlgorithm": "sec:digestAlgorithm", + "digestValue": "sec:digestValue", + "domain": "sec:domain", + "expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"}, + "initializationVector": "sec:initializationVector", + "member": {"@id": "schema:member", "@type": "@id"}, + "memberOf": {"@id": "schema:memberOf", "@type": "@id"}, + "nonce": "sec:nonce", + "normalizationAlgorithm": "sec:normalizationAlgorithm", + "owner": {"@id": "sec:owner", "@type": "@id"}, + "password": "sec:password", + "privateKey": {"@id": "sec:privateKey", "@type": "@id"}, + "privateKeyPem": "sec:privateKeyPem", + "publicKey": {"@id": "sec:publicKey", "@type": "@id"}, + "publicKeyPem": "sec:publicKeyPem", + "publicKeyService": { + "@id": "sec:publicKeyService", "@type": "@id" + }, + "revoked": {"@id": "sec:revoked", "@type": "xsd:dateTime"}, + "signature": "sec:signature", + "signatureAlgorithm": "sec:signatureAlgorithm", + "signatureValue": "sec:signatureValue", + "CryptographicKey": "sec:Key", + "EncryptedMessage": "sec:EncryptedMessage", + "GraphSignature2012": "sec:GraphSignature2012", + "LinkedDataSignature2015": "sec:LinkedDataSignature2015", + "accessControl": {"@id": "perm:accessControl", "@type": "@id"}, + "writePermission": {"@id": "perm:writePermission", "@type": "@id"} + } + } + + +def getActivitystreamsSchema() -> {}: + return { + "@context": { + "@vocab": "_:", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "as": "https://www.w3.org/ns/activitystreams#", + "ldp": "http://www.w3.org/ns/ldp#", + "vcard": "http://www.w3.org/2006/vcard/ns#", + "id": "@id", + "type": "@type", + "Accept": "as:Accept", + "Activity": "as:Activity", + "IntransitiveActivity": "as:IntransitiveActivity", + "Add": "as:Add", + "Announce": "as:Announce", + "Application": "as:Application", + "Arrive": "as:Arrive", + "Article": "as:Article", + "Audio": "as:Audio", + "Block": "as:Block", + "Collection": "as:Collection", + "CollectionPage": "as:CollectionPage", + "Relationship": "as:Relationship", + "Create": "as:Create", + "Delete": "as:Delete", + "Dislike": "as:Dislike", + "Document": "as:Document", + "Event": "as:Event", + "Follow": "as:Follow", + "Flag": "as:Flag", + "Group": "as:Group", + "Ignore": "as:Ignore", + "Image": "as:Image", + "Invite": "as:Invite", + "Join": "as:Join", + "Leave": "as:Leave", + "Like": "as:Like", + "Link": "as:Link", + "Mention": "as:Mention", + "Note": "as:Note", + "Object": "as:Object", + "Offer": "as:Offer", + "OrderedCollection": "as:OrderedCollection", + "OrderedCollectionPage": "as:OrderedCollectionPage", + "Organization": "as:Organization", + "Page": "as:Page", + "Person": "as:Person", + "Place": "as:Place", + "Profile": "as:Profile", + "Question": "as:Question", + "Reject": "as:Reject", + "Remove": "as:Remove", + "Service": "as:Service", + "TentativeAccept": "as:TentativeAccept", + "TentativeReject": "as:TentativeReject", + "Tombstone": "as:Tombstone", + "Undo": "as:Undo", + "Update": "as:Update", + "Video": "as:Video", + "View": "as:View", + "Listen": "as:Listen", + "Read": "as:Read", + "Move": "as:Move", + "Travel": "as:Travel", + "IsFollowing": "as:IsFollowing", + "IsFollowedBy": "as:IsFollowedBy", + "IsContact": "as:IsContact", + "IsMember": "as:IsMember", + "subject": { + "@id": "as:subject", + "@type": "@id" + }, + "relationship": { + "@id": "as:relationship", + "@type": "@id" + }, + "actor": { + "@id": "as:actor", + "@type": "@id" + }, + "attributedTo": { + "@id": "as:attributedTo", + "@type": "@id" + }, + "attachment": { + "@id": "as:attachment", + "@type": "@id" + }, + "bcc": { + "@id": "as:bcc", + "@type": "@id" + }, + "bto": { + "@id": "as:bto", + "@type": "@id" + }, + "cc": { + "@id": "as:cc", + "@type": "@id" + }, + "context": { + "@id": "as:context", + "@type": "@id" + }, + "current": { + "@id": "as:current", + "@type": "@id" + }, + "first": { + "@id": "as:first", + "@type": "@id" + }, + "generator": { + "@id": "as:generator", + "@type": "@id" + }, + "icon": { + "@id": "as:icon", + "@type": "@id" + }, + "image": { + "@id": "as:image", + "@type": "@id" + }, + "inReplyTo": { + "@id": "as:inReplyTo", + "@type": "@id" + }, + "items": { + "@id": "as:items", + "@type": "@id" + }, + "instrument": { + "@id": "as:instrument", + "@type": "@id" + }, + "orderedItems": { + "@id": "as:items", + "@type": "@id", + "@container": "@list" + }, + "last": { + "@id": "as:last", + "@type": "@id" + }, + "location": { + "@id": "as:location", + "@type": "@id" + }, + "next": { + "@id": "as:next", + "@type": "@id" + }, + "object": { + "@id": "as:object", + "@type": "@id" + }, + "oneOf": { + "@id": "as:oneOf", + "@type": "@id" + }, + "anyOf": { + "@id": "as:anyOf", + "@type": "@id" + }, + "closed": { + "@id": "as:closed", + "@type": "xsd:dateTime" + }, + "origin": { + "@id": "as:origin", + "@type": "@id" + }, + "accuracy": { + "@id": "as:accuracy", + "@type": "xsd:float" + }, + "prev": { + "@id": "as:prev", + "@type": "@id" + }, + "preview": { + "@id": "as:preview", + "@type": "@id" + }, + "replies": { + "@id": "as:replies", + "@type": "@id" + }, + "result": { + "@id": "as:result", + "@type": "@id" + }, + "audience": { + "@id": "as:audience", + "@type": "@id" + }, + "partOf": { + "@id": "as:partOf", + "@type": "@id" + }, + "tag": { + "@id": "as:tag", + "@type": "@id" + }, + "target": { + "@id": "as:target", + "@type": "@id" + }, + "to": { + "@id": "as:to", + "@type": "@id" + }, + "url": { + "@id": "as:url", + "@type": "@id" + }, + "altitude": { + "@id": "as:altitude", + "@type": "xsd:float" + }, + "content": "as:content", + "contentMap": { + "@id": "as:content", + "@container": "@language" + }, + "name": "as:name", + "nameMap": { + "@id": "as:name", + "@container": "@language" + }, + "duration": { + "@id": "as:duration", + "@type": "xsd:duration" + }, + "endTime": { + "@id": "as:endTime", + "@type": "xsd:dateTime" + }, + "height": { + "@id": "as:height", + "@type": "xsd:nonNegativeInteger" + }, + "href": { + "@id": "as:href", + "@type": "@id" + }, + "hreflang": "as:hreflang", + "latitude": { + "@id": "as:latitude", + "@type": "xsd:float" + }, + "longitude": { + "@id": "as:longitude", + "@type": "xsd:float" + }, + "mediaType": "as:mediaType", + "published": { + "@id": "as:published", + "@type": "xsd:dateTime" + }, + "radius": { + "@id": "as:radius", + "@type": "xsd:float" + }, + "rel": "as:rel", + "startIndex": { + "@id": "as:startIndex", + "@type": "xsd:nonNegativeInteger" + }, + "startTime": { + "@id": "as:startTime", + "@type": "xsd:dateTime" + }, + "summary": "as:summary", + "summaryMap": { + "@id": "as:summary", + "@container": "@language" + }, + "totalItems": { + "@id": "as:totalItems", + "@type": "xsd:nonNegativeInteger" + }, + "units": "as:units", + "updated": { + "@id": "as:updated", + "@type": "xsd:dateTime" + }, + "width": { + "@id": "as:width", + "@type": "xsd:nonNegativeInteger" + }, + "describes": { + "@id": "as:describes", + "@type": "@id" + }, + "formerType": { + "@id": "as:formerType", + "@type": "@id" + }, + "deleted": { + "@id": "as:deleted", + "@type": "xsd:dateTime" + }, + "inbox": { + "@id": "ldp:inbox", + "@type": "@id" + }, + "outbox": { + "@id": "as:outbox", + "@type": "@id" + }, + "following": { + "@id": "as:following", + "@type": "@id" + }, + "followers": { + "@id": "as:followers", + "@type": "@id" + }, + "streams": { + "@id": "as:streams", + "@type": "@id" + }, + "preferredUsername": "as:preferredUsername", + "endpoints": { + "@id": "as:endpoints", + "@type": "@id" + }, + "uploadMedia": { + "@id": "as:uploadMedia", + "@type": "@id" + }, + "proxyUrl": { + "@id": "as:proxyUrl", + "@type": "@id" + }, + "liked": { + "@id": "as:liked", + "@type": "@id" + }, + "oauthAuthorizationEndpoint": { + "@id": "as:oauthAuthorizationEndpoint", + "@type": "@id" + }, + "oauthTokenEndpoint": { + "@id": "as:oauthTokenEndpoint", + "@type": "@id" + }, + "provideClientKey": { + "@id": "as:provideClientKey", + "@type": "@id" + }, + "signClientKey": { + "@id": "as:signClientKey", + "@type": "@id" + }, + "sharedInbox": { + "@id": "as:sharedInbox", + "@type": "@id" + }, + "Public": { + "@id": "as:Public", + "@type": "@id" + }, + "source": "as:source", + "likes": { + "@id": "as:likes", + "@type": "@id" + }, + "shares": { + "@id": "as:shares", + "@type": "@id" + }, + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + } + } + } + + def load_document(url): """ Retrieves JSON-LD at the given URL. @@ -380,6 +850,22 @@ def load_document(url): url_opener.addheaders = [ ('Accept', 'application/ld+json, application/json'), ('Accept-Encoding', 'deflate')] + + if url == 'https://w3id.org/identity/v1': + doc = { + 'contextUrl': None, + 'documentUrl': url, + 'document': getV1Schema() + } + return doc + elif url == 'https://www.w3.org/ns/activitystreams': + doc = { + 'contextUrl': None, + 'documentUrl': url, + 'document': getActivitystreamsSchema() + } + return doc + with closing(url_opener.open(url)) as handle: if handle.info().get('Content-Encoding') == 'gzip': buf = io.BytesIO(handle.read()) diff --git a/tests.py b/tests.py index bc9003482..cd957bf83 100644 --- a/tests.py +++ b/tests.py @@ -86,8 +86,8 @@ from content import replaceContentDuplicates from content import removeTextFormatting from content import removeHtmlTag from theme import setCSSparam -from jsonldsig import testSignJsonld -from jsonldsig import jsonldVerify +from linked_data_sig import generateJsonSignature +from linked_data_sig import verifyJsonSignature from newsdaemon import hashtagRuleTree from newsdaemon import hashtagRuleResolve from newswire import getNewswireTags @@ -1977,8 +1977,10 @@ def testRemoveTextFormatting(): def testJsonld(): print("testJsonld") + jldDocument = { "@context": "https://www.w3.org/ns/activitystreams", + "actor": "https://somesite.net/users/gerbil", "description": "My json document", "numberField": 83582, "object": { @@ -2023,27 +2025,32 @@ def testJsonld(): 'TwIDAQAB\n' \ '-----END PUBLIC KEY-----' - signedDocument = testSignJsonld(jldDocument, privateKeyPem) + signedDocument = jldDocument.copy() + generateJsonSignature(signedDocument, privateKeyPem) assert(signedDocument) assert(signedDocument.get('signature')) assert(signedDocument['signature'].get('signatureValue')) assert(signedDocument['signature'].get('type')) assert(len(signedDocument['signature']['signatureValue']) > 50) - assert(signedDocument['signature']['type'] == 'RsaSignatureSuite2017') - assert(jsonldVerify(signedDocument, publicKeyPem)) + # print(str(signedDocument['signature'])) + assert(signedDocument['signature']['type'] == 'RsaSignature2017') + assert(verifyJsonSignature(signedDocument, publicKeyPem)) + # alter the signed document signedDocument['object']['content'] = 'forged content' - assert(not jsonldVerify(signedDocument, publicKeyPem)) + assert(not verifyJsonSignature(signedDocument, publicKeyPem)) jldDocument2 = { "@context": "https://www.w3.org/ns/activitystreams", + "actor": "https://somesite.net/users/gerbil", "description": "Another json document", "numberField": 13353, "object": { "content": "More content" } } - signedDocument2 = testSignJsonld(jldDocument2, privateKeyPem) + signedDocument2 = jldDocument2.copy() + generateJsonSignature(signedDocument2, privateKeyPem) assert(signedDocument2) assert(signedDocument2.get('signature')) assert(signedDocument2['signature'].get('signatureValue')) @@ -2051,6 +2058,8 @@ def testJsonld(): if signedDocument['signature']['signatureValue'] == \ signedDocument2['signature']['signatureValue']: print('json signature has not changed for different documents') + assert '.' not in str(signedDocument['signature']['signatureValue']) + assert len(str(signedDocument['signature']['signatureValue'])) > 340 assert(signedDocument['signature']['signatureValue'] != signedDocument2['signature']['signatureValue']) From 17fd7db5f34a1b1d0f55e0bdc3aaf708be07c33a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 4 Jan 2021 19:03:20 +0000 Subject: [PATCH 25/28] Comment --- pyjsonld.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyjsonld.py b/pyjsonld.py index 5b178a56f..531c1d9da 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -444,6 +444,7 @@ def getV1Schema() -> {}: def getActivitystreamsSchema() -> {}: + # https://www.w3.org/ns/activitystreams return { "@context": { "@vocab": "_:", From 11d7192edb900345e359f4de787422a22d91cf58 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 4 Jan 2021 19:09:12 +0000 Subject: [PATCH 26/28] Don't download schemas from remote sites --- pyjsonld.py | 42 +----------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/pyjsonld.py b/pyjsonld.py index 531c1d9da..0a183c305 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -24,9 +24,7 @@ __all__ = [ 'JsonLdProcessor', 'JsonLdError', 'ActiveContextCache'] import copy -import gzip import hashlib -import io import json import os import posixpath @@ -37,7 +35,6 @@ import string import sys import traceback from collections import deque, namedtuple -from contextlib import closing from numbers import Integral, Real try: @@ -77,7 +74,6 @@ except ImportError: # support python 2 if sys.version_info[0] >= 3: - from urllib.request import build_opener as urllib_build_opener from urllib.request import HTTPSHandler import urllib.parse as urllib_parse from http.client import HTTPSConnection @@ -86,7 +82,6 @@ if sys.version_info[0] >= 3: def cmp(a, b): return (a > b) - (a < b) else: - from urllib2 import build_opener as urllib_build_opener from urllib2 import HTTPSHandler import urlparse as urllib_parse from httplib import HTTPSConnection @@ -846,11 +841,6 @@ def load_document(url): 'URLs are supported.', 'jsonld.InvalidUrl', {'url': url}, code='loading document failed') - https_handler = VerifiedHTTPSHandler() - url_opener = urllib_build_opener(https_handler) - url_opener.addheaders = [ - ('Accept', 'application/ld+json, application/json'), - ('Accept-Encoding', 'deflate')] if url == 'https://w3id.org/identity/v1': doc = { @@ -866,37 +856,7 @@ def load_document(url): 'document': getActivitystreamsSchema() } return doc - - with closing(url_opener.open(url)) as handle: - if handle.info().get('Content-Encoding') == 'gzip': - buf = io.BytesIO(handle.read()) - f = gzip.GzipFile(fileobj=buf, mode='rb') - data = f.read() - else: - data = handle.read() - doc = { - 'contextUrl': None, - 'documentUrl': url, - 'document': data.decode('utf8') - } - doc['documentUrl'] = handle.geturl() - headers = dict(handle.info()) - content_type = headers.get('content-type') - link_header = headers.get('link') - if link_header and content_type != 'application/ld+json': - link_header = parse_link_header(link_header).get( - LINK_HEADER_REL) - # only 1 related link header permitted - if isinstance(link_header, list): - raise JsonLdError( - 'URL could not be dereferenced, it has more than one ' - 'associated HTTP Link Header.', - 'jsonld.LoadDocumentError', - {'url': url}, - code='multiple context link headers') - if link_header: - doc['contextUrl'] = link_header['target'] - return doc + return None except JsonLdError as e: raise e except Exception as cause: From 3837ee2536b5649b9806301374feab08248d7218 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 4 Jan 2021 19:11:17 +0000 Subject: [PATCH 27/28] support dat --- pyjsonld.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyjsonld.py b/pyjsonld.py index 0a183c305..1826cf82f 100644 --- a/pyjsonld.py +++ b/pyjsonld.py @@ -833,11 +833,11 @@ def load_document(url): # validate URL pieces = urllib_parse.urlparse(url) if (not all([pieces.scheme, pieces.netloc]) or - pieces.scheme not in ['http', 'https'] or + pieces.scheme not in ['http', 'https', 'dat'] or set(pieces.netloc) > set( string.ascii_letters + string.digits + '-.:')): raise JsonLdError( - 'URL could not be dereferenced; only "http" and "https" ' + 'URL could not be dereferenced; only http/https/dat ' 'URLs are supported.', 'jsonld.InvalidUrl', {'url': url}, code='loading document failed') From b8a698d887495b2426aded9141cea1ee02ce4210 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Mon, 4 Jan 2021 20:26:55 +0000 Subject: [PATCH 28/28] Show keyId for json signature check --- inbox.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/inbox.py b/inbox.py index 6ad5daecf..08fefc91c 100644 --- a/inbox.py +++ b/inbox.py @@ -2720,16 +2720,20 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, # use the original json message received, not one which may have # been modified along the way if not verifyJsonSignature(queueJson['original'], pubKey): - print('WARN: jsonld inbox signature check failed ' + - keyId + ' ' + pubKey + ' ' + - str(queueJson['original'])) + if debug: + print('WARN: jsonld inbox signature check failed ' + + keyId + ' ' + pubKey + ' ' + + str(queueJson['original'])) + else: + print('WARN: jsonld inbox signature check failed ' + + keyId) if os.path.isfile(queueFilename): os.remove(queueFilename) if len(queue) > 0: queue.pop(0) continue else: - print('jsonld inbox signature check success') + print('jsonld inbox signature check success ' + keyId) # set the id to the same as the post filename # This makes the filename and the id consistent