diff --git a/Makefile b/Makefile index 45ef03d96..789fdbf43 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ source: rm -f ../${APP}*.deb ../${APP}*.changes ../${APP}*.asc ../${APP}*.dsc cd .. && mv ${APP} ${APP}-${VERSION} && tar -zcvf ${APP}_${VERSION}.orig.tar.gz ${APP}-${VERSION}/ && mv ${APP}-${VERSION} ${APP} clean: - rm -f *.*~ *~ + rm -f *.*~ *~ *.dot rm -f orgs/*~ rm -f website/EN/*~ rm -f gemini/EN/*~ diff --git a/README.md b/README.md index 50223b2b8..ecc98bf50 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ On Arch/Parabola: sudo pacman -S tor python-pip python-pysocks python-pycryptodome \ imagemagick python-pillow python-requests \ perl-image-exiftool python-numpy python-dateutil \ - certbot flake8 + certbot flake8 bandit sudo pip3 install pyLD pyqrcode pypng ``` @@ -41,7 +41,8 @@ sudo apt install -y \ python3-idna python3-requests \ python3-pyld python3-django-timezone-field \ libimage-exiftool-perl python3-flake8 \ - python3-pyqrcode python3-png certbot nginx + python3-pyqrcode python3-png python3-bandit \ + certbot nginx ``` ## Installation @@ -200,6 +201,16 @@ Static analysis can be run with: ./static_analysis ``` +## Running a security audit + +To run a security audit: + +``` bash +./security_audit +``` + +Note that not all of the issues identified will necessarily be relevant to this project. + ## Installing on Onion or i2p domains If you don't have access to the clearnet, or prefer not to use it, then it's possible to run an Epicyon instance easily from your laptop. There are scripts within the ```deploy``` directory which can be used to install an instance on a Debian or Arch/Parabola operating system. With some modification of package names they could be also used with other distros. diff --git a/README_commandline.md b/README_commandline.md index 2fbcff053..4bce92a8c 100644 --- a/README_commandline.md +++ b/README_commandline.md @@ -136,6 +136,23 @@ If you want to view the raw json: python3 epicyon.py --postsraw nickname@domain ``` +## Listing referenced domains + +To list the domains referenced in public posts: + +``` bash +python3 epicyon.py --postDomains nickname@domain +``` + +## Plotting federated instances + +To plot a set of federated instances, based upon a sample of handles on those instances: + +``` bash +python3 epicyon.py --socnet nickname1@domain1,nickname2@domain2,nickname3@domain3 +xdot socnet.dot +``` + ## Delete posts To delete a post which you wrote you must first know its url. It is usually something like: diff --git a/auth.py b/auth.py index 8f162e429..8297aa816 100644 --- a/auth.py +++ b/auth.py @@ -10,7 +10,7 @@ import base64 import hashlib import binascii import os -import random +import secrets def hashPassword(password: str) -> str: @@ -162,4 +162,4 @@ def authorize(baseDir: str, path: str, authHeader: str, debug: bool) -> bool: def createPassword(length=10): validChars = 'abcdefghijklmnopqrstuvwxyz' + \ 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - return ''.join((random.choice(validChars) for i in range(length))) + return ''.join((secrets.choice(validChars) for i in range(length))) diff --git a/content.py b/content.py index 6fae45675..bc3c9af4a 100644 --- a/content.py +++ b/content.py @@ -14,6 +14,32 @@ from utils import fileLastModified from utils import getLinkPrefixes +def dangerousMarkup(content: str) -> bool: + """Returns true if the given content contains dangerous html markup + """ + if '<' not in content: + return False + if '>' not in content: + return False + contentSections = content.split('<') + invalidStrings = ('script', 'canvas', 'style', 'abbr', + 'frame', 'iframe', 'html', 'body', + 'hr') + for markup in contentSections: + if '>' not in markup: + continue + markup = markup.split('>')[0].strip() + if ' ' not in markup: + for badStr in invalidStrings: + if badStr in markup: + return True + else: + for badStr in invalidStrings: + if badStr + ' ' in markup: + return True + return False + + def switchWords(baseDir: str, nickname: str, domain: str, content: str) -> str: """Performs word replacements. eg. Trump -> The Orange Menace """ @@ -400,6 +426,24 @@ def removeTextFormatting(content: str) -> str: return content +def removeHtml(content: str) -> str: + """Removes html links from the given content. + Used to ensure that profile descriptions don't contain dubious content + """ + if '<' not in content: + return content + removing = False + result = '' + for ch in content: + if ch == '<': + removing = True + elif ch == '>': + removing = False + elif not removing: + result += ch + return result + + def removeLongWords(content: str, maxWordLength: int, longWordsList: []) -> str: """Breaks up long words so that on mobile screens this doesn't diff --git a/daemon.py b/daemon.py index df1dd3ba5..152285641 100644 --- a/daemon.py +++ b/daemon.py @@ -30,7 +30,9 @@ from metadata import metaDataNodeInfo from pgp import getEmailAddress from pgp import setEmailAddress from pgp import getPGPpubKey +from pgp import getPGPfingerprint from pgp import setPGPpubKey +from pgp import setPGPfingerprint from xmpp import getXmppAddress from xmpp import setXmppAddress from ssb import getSSBAddress @@ -177,6 +179,8 @@ from cache import getPersonFromCache from httpsig import verifyPostHeaders from theme import setTheme from theme import getTheme +from theme import enableGrayscale +from theme import disableGrayscale from schedule import runPostSchedule from schedule import runPostScheduleWatchdog from schedule import removeScheduledPosts @@ -533,7 +537,7 @@ class PubServer(BaseHTTPRequestHandler): except BaseException: pass if not etag: - etag = sha1(data).hexdigest() + etag = sha1(data).hexdigest() # nosec try: with open(mediaFilename + '.etag', 'w') as etagFile: etagFile.write(etag) @@ -1549,6 +1553,7 @@ class PubServer(BaseHTTPRequestHandler): optionsLink = optionsList[3] donateUrl = None PGPpubKey = None + PGPfingerprint = None xmppAddress = None matrixAddress = None blogAddress = None @@ -1567,6 +1572,7 @@ class PubServer(BaseHTTPRequestHandler): toxAddress = getToxAddress(actorJson) emailAddress = getEmailAddress(actorJson) PGPpubKey = getPGPpubKey(actorJson) + PGPfingerprint = getPGPfingerprint(actorJson) msg = htmlPersonOptions(self.server.translate, self.server.baseDir, self.server.domain, @@ -1577,7 +1583,8 @@ class PubServer(BaseHTTPRequestHandler): pageNumber, donateUrl, xmppAddress, matrixAddress, ssbAddress, blogAddress, - toxAddress, PGPpubKey, + toxAddress, + PGPpubKey, PGPfingerprint, emailAddress).encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -5093,7 +5100,7 @@ class PubServer(BaseHTTPRequestHandler): else: with open(mediaFilename, 'rb') as avFile: mediaBinary = avFile.read() - etag = sha1(mediaBinary).hexdigest() + etag = sha1(mediaBinary).hexdigest() # nosec try: with open(mediaTagFilename, 'w') as etagFile: etagFile.write(etag) @@ -6242,6 +6249,17 @@ class PubServer(BaseHTTPRequestHandler): setPGPpubKey(actorJson, '') actorChanged = True + currentPGPfingerprint = getPGPfingerprint(actorJson) + if fields.get('openpgp'): + if fields['openpgp'] != currentPGPfingerprint: + setPGPfingerprint(actorJson, + fields['openpgp']) + actorChanged = True + else: + if currentPGPfingerprint: + setPGPfingerprint(actorJson, '') + actorChanged = True + currentDonateUrl = getDonationUrl(actorJson) if fields.get('donateUrl'): if fields['donateUrl'] != currentDonateUrl: @@ -6478,6 +6496,14 @@ class PubServer(BaseHTTPRequestHandler): if actorJson['type'] != 'Person': actorJson['type'] = 'Person' actorChanged = True + grayscale = False + if fields.get('grayscale'): + if fields['grayscale'] == 'on': + grayscale = True + if grayscale: + enableGrayscale(self.server.baseDir) + else: + disableGrayscale(self.server.baseDir) # save filtered words list filterFilename = \ self.server.baseDir + '/accounts/' + \ diff --git a/deploy/i2p b/deploy/i2p index d390d5500..2afeb74ab 100755 --- a/deploy/i2p +++ b/deploy/i2p @@ -64,7 +64,7 @@ if [ -f /usr/bin/pacman ]; then imagemagick python-pillow python-requests \ perl-image-exiftool python-numpy python-dateutil \ certbot flake8 git i2pd wget qrencode \ - proxychains midori + proxychains midori bandit pip3 install pyLD pyqrcode pypng else apt-get update @@ -75,7 +75,7 @@ else libimage-exiftool-perl python3-flake8 python3-pyld \ python3-django-timezone-field nginx git i2pd wget \ python3-pyqrcode qrencode python3-png \ - proxychains midori + proxychains midori python3-bandit fi if [ ! -d /etc/i2pd ]; then diff --git a/deploy/onion b/deploy/onion index 6c1093c04..c60df40b1 100755 --- a/deploy/onion +++ b/deploy/onion @@ -38,7 +38,7 @@ if [ -f /usr/bin/pacman ]; then pacman -S --noconfirm tor python-pip python-pysocks python-pycryptodome \ imagemagick python-pillow python-requests \ perl-image-exiftool python-numpy python-dateutil \ - certbot flake8 git qrencode + certbot flake8 git qrencode bandit pip3 install pyLD pyqrcode pypng else apt-get update @@ -48,7 +48,7 @@ else python3-setuptools python3-socks python3-idna \ libimage-exiftool-perl python3-flake8 python3-pyld \ python3-django-timezone-field tor nginx git qrencode \ - python3-pyqrcode python3-png + python3-pyqrcode python3-png python3-bandit fi echo 'Cloning the epicyon repo' diff --git a/donate.py b/donate.py index 48298a9f7..db68f9680 100644 --- a/donate.py +++ b/donate.py @@ -41,6 +41,14 @@ def getDonationUrl(actorJson: {}) -> str: def setDonationUrl(actorJson: {}, donateUrl: str) -> None: """Sets a link used for donations """ + notUrl = False + if '.' not in donateUrl: + notUrl = True + if '://' not in donateUrl: + notUrl = True + if ' ' in donateUrl: + notUrl = True + if not actorJson.get('attachment'): actorJson['attachment'] = [] @@ -65,6 +73,8 @@ def setDonationUrl(actorJson: {}, donateUrl: str) -> None: break if propertyFound: actorJson['attachment'].remove(propertyFound) + if notUrl: + return donateValue = \ ' "$epicyonLikeFile" + chown ${PROJECT_NAME}:${PROJECT_NAME} "$epicyonLkeFile" + fi + fi + # send notifications for replies to XMPP/email users epicyonReplyFile="$epicyonDir/.newReply" if [ -f "$epicyonReplyFile" ]; then diff --git a/epicyon.py b/epicyon.py index 663b5064c..791c1e1cf 100644 --- a/epicyon.py +++ b/epicyon.py @@ -15,6 +15,7 @@ from person import deactivateAccount from skills import setSkillLevel from roles import setRole from webfinger import webfingerHandle +from posts import getPublicPostDomains from posts import sendBlockViaServer from posts import sendUndoBlockViaServer from posts import createPublicPost @@ -66,6 +67,7 @@ from shares import sendUndoShareViaServer from shares import addShare from theme import setTheme from announce import sendAnnounceViaServer +from socnet import instancesGraph import argparse @@ -146,6 +148,14 @@ parser.add_argument('--actor', dest='actor', type=str, parser.add_argument('--posts', dest='posts', type=str, default=None, help='Show posts for the given handle') +parser.add_argument('--postDomains', dest='postDomains', type=str, + default=None, + help='Show domains referenced in public ' + 'posts for the given handle') +parser.add_argument('--socnet', dest='socnet', type=str, + default=None, + help='Show dot diagram for social network ' + 'of federated instances') parser.add_argument('--postsraw', dest='postsraw', type=str, default=None, help='Show raw json of posts for the given handle') @@ -386,8 +396,16 @@ if baseDir.endswith('/'): if args.posts: if '@' not in args.posts: - print('Syntax: --posts nickname@domain') - sys.exit() + if '/users/' in args.posts: + postsNickname = getNicknameFromActor(args.posts) + postsDomain, postsPort = getDomainFromActor(args.posts) + args.posts = postsNickname + '@' + postsDomain + if postsPort: + if postsPort != 80 and postsPort != 443: + args.posts += ':' + str(postsPort) + else: + print('Syntax: --posts nickname@domain') + sys.exit() if not args.http: args.port = 443 nickname = args.posts.split('@')[0] @@ -395,8 +413,12 @@ if args.posts: proxyType = None if args.tor or domain.endswith('.onion'): proxyType = 'tor' + if domain.endswith('.onion'): + args.port = 80 elif args.i2p or domain.endswith('.i2p'): proxyType = 'i2p' + if domain.endswith('.i2p'): + args.port = 80 elif args.gnunet: proxyType = 'gnunet' getPublicPostsOfPerson(baseDir, nickname, domain, False, True, @@ -404,6 +426,63 @@ if args.posts: __version__) sys.exit() +if args.postDomains: + if '@' not in args.postDomains: + if '/users/' in args.postDomains: + postsNickname = getNicknameFromActor(args.posts) + postsDomain, postsPort = getDomainFromActor(args.posts) + args.postDomains = postsNickname + '@' + postsDomain + if postsPort: + if postsPort != 80 and postsPort != 443: + args.postDomains += ':' + str(postsPort) + else: + print('Syntax: --postDomains nickname@domain') + sys.exit() + if not args.http: + args.port = 443 + nickname = args.postDomains.split('@')[0] + domain = args.postDomains.split('@')[1] + proxyType = None + if args.tor or domain.endswith('.onion'): + proxyType = 'tor' + if domain.endswith('.onion'): + args.port = 80 + elif args.i2p or domain.endswith('.i2p'): + proxyType = 'i2p' + if domain.endswith('.i2p'): + args.port = 80 + elif args.gnunet: + proxyType = 'gnunet' + domainList = [] + domainList = getPublicPostDomains(baseDir, nickname, domain, + proxyType, args.port, + httpPrefix, debug, + __version__, domainList) + for postDomain in domainList: + print(postDomain) + sys.exit() + +if args.socnet: + if ',' not in args.socnet: + print('Syntax: ' + '--socnet nick1@domain1,nick2@domain2,nick3@domain3') + sys.exit() + + if not args.http: + args.port = 443 + proxyType = 'tor' + dotGraph = instancesGraph(baseDir, args.socnet, + proxyType, args.port, + httpPrefix, debug, + __version__) + try: + with open('socnet.dot', 'w') as fp: + fp.write(dotGraph) + print('Saved to socnet.dot') + except BaseException: + pass + sys.exit() + if args.postsraw: if '@' not in args.postsraw: print('Syntax: --postsraw nickname@domain') diff --git a/gemini/EN/install.gmi b/gemini/EN/install.gmi index 6484b30fa..0eb54ba71 100644 --- a/gemini/EN/install.gmi +++ b/gemini/EN/install.gmi @@ -4,7 +4,7 @@ You will need python version 3.7 or later. On a Debian based system: - sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-pyld python3-django-timezone-field python3-pyqrcode python3-png libimage-exiftool-perl certbot nginx + sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-pyld python3-django-timezone-field python3-pyqrcode python3-png python3-bandit libimage-exiftool-perl certbot nginx The following instructions install Epicyon to the /opt directory. It's not essential that it be installed there, and it could be in any other preferred directory. diff --git a/img/eyes.jpg b/img/eyes.jpg new file mode 100644 index 000000000..e6a304847 Binary files /dev/null and b/img/eyes.jpg differ diff --git a/inbox.py b/inbox.py index 56c3b5f80..20496828e 100644 --- a/inbox.py +++ b/inbox.py @@ -63,6 +63,7 @@ from media import replaceYouTube from git import isGitPatch from git import receiveGitPatch from followingCalendar import receivingCalendarEvents +from content import dangerousMarkup def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -981,6 +982,7 @@ def receiveUpdate(recentPostsCache: {}, session, baseDir: str, def receiveLike(recentPostsCache: {}, session, handle: str, isGroup: bool, baseDir: str, httpPrefix: str, domain: str, port: int, + onionDomain: str, sendThreads: [], postLog: [], cachedWebfingers: {}, personCache: {}, messageJson: {}, federationList: [], debug: bool) -> bool: @@ -1033,6 +1035,8 @@ def receiveLike(recentPostsCache: {}, updateLikesCollection(recentPostsCache, baseDir, postFilename, messageJson['object'], messageJson['actor'], domain, debug) + likeNotify(baseDir, domain, onionDomain, handle, + messageJson['actor'], messageJson['object']) return True @@ -1596,22 +1600,20 @@ def validPostContent(baseDir: str, nickname: str, domain: str, return False if 'Z' not in messageJson['object']['published']: return False + if isGitPatch(baseDir, nickname, domain, messageJson['object']['type'], messageJson['object']['summary'], messageJson['object']['content']): return True - # check for bad html - invalidStrings = ('', '', - '', '', - '', '', '
', '
') - for badStr in invalidStrings: - if badStr in messageJson['object']['content']: - if messageJson['object'].get('id'): - print('REJECT ARBITRARY HTML: ' + messageJson['object']['id']) - print('REJECT ARBITRARY HTML: bad string in post - ' + - messageJson['object']['content']) - return False + + if dangerousMarkup(messageJson['object']['content']): + if messageJson['object'].get('id'): + print('REJECT ARBITRARY HTML: ' + messageJson['object']['id']) + print('REJECT ARBITRARY HTML: bad string in post - ' + + messageJson['object']['content']) + return False + # check (rough) number of mentions mentionsEst = estimateNumberOfMentions(messageJson['object']['content']) if mentionsEst > maxMentions: @@ -1704,6 +1706,54 @@ def dmNotify(baseDir: str, handle: str, url: str) -> None: fp.write(url) +def likeNotify(baseDir: str, domain: str, onionDomain: str, + handle: str, actor: str, url: str) -> None: + """Creates a notification that a like has arrived + """ + # This is not you liking your own post + if actor in url: + return + + # check that the liked post was by this handle + nickname = handle.split('@')[0] + if '/' + domain + '/users/' + nickname not in url: + if not onionDomain: + return + if '/' + onionDomain + '/users/' + nickname not in url: + return + + accountDir = baseDir + '/accounts/' + handle + if not os.path.isdir(accountDir): + return + likeFile = accountDir + '/.newLike' + if os.path.isfile(likeFile): + if '##sent##' not in open(likeFile).read(): + return + + likerNickname = getNicknameFromActor(actor) + likerDomain, likerPort = getDomainFromActor(actor) + if likerNickname and likerDomain: + likerHandle = likerNickname + '@' + likerDomain + else: + print('likeNotify likerHandle: ' + + str(likerNickname) + '@' + str(likerDomain)) + likerHandle = actor + if likerHandle != handle: + likeStr = likerHandle + ' ' + url + prevLikeFile = accountDir + '/.prevLike' + # was there a previous like notification? + if os.path.isfile(prevLikeFile): + # is it the same as the current notification ? + with open(prevLikeFile, 'r') as likeFile: + prevLikeStr = likeFile.read() + if prevLikeStr == likeStr: + return + with open(prevLikeFile, 'w') as fp: + fp.write(likeStr) + with open(likeFile, 'w') as fp: + fp.write(likeStr) + + def replyNotify(baseDir: str, handle: str, url: str) -> None: """Creates a notification that a new reply has arrived """ @@ -1970,6 +2020,7 @@ def inboxAfterCapabilities(recentPostsCache: {}, maxRecentPosts: int, session, handle, isGroup, baseDir, httpPrefix, domain, port, + onionDomain, sendThreads, postLog, cachedWebfingers, personCache, diff --git a/media.py b/media.py index fd372b760..4afa96571 100644 --- a/media.py +++ b/media.py @@ -38,12 +38,15 @@ def removeMetaData(imageFilename: str, outputFilename: str) -> None: so better to use a dedicated tool if one is installed """ copyfile(imageFilename, outputFilename) + if not os.path.isfile(outputFilename): + print('ERROR: unable to remove metadata from ' + imageFilename) + return if os.path.isfile('/usr/bin/exiftool'): print('Removing metadata from ' + outputFilename + ' using exiftool') - os.system('exiftool -all= ' + outputFilename) + os.system('exiftool -all= ' + outputFilename) # nosec elif os.path.isfile('/usr/bin/mogrify'): print('Removing metadata from ' + outputFilename + ' using mogrify') - os.system('/usr/bin/mogrify -strip '+outputFilename) + os.system('/usr/bin/mogrify -strip ' + outputFilename) # nosec def getImageHash(imageFilename: str) -> str: @@ -116,7 +119,7 @@ def updateEtag(mediaFilename: str) -> None: if not data: return # calculate hash - etag = sha1(data).hexdigest() + etag = sha1(data).hexdigest() # nosec # save the hash try: with open(mediaFilename + '.etag', 'w') as etagFile: diff --git a/person.py b/person.py index 3db66c57b..2196b68b3 100644 --- a/person.py +++ b/person.py @@ -151,14 +151,16 @@ def randomizeActorImages(personJson: {}) -> None: personId = personJson['id'] lastPartOfFilename = personJson['icon']['url'].split('/')[-1] existingExtension = lastPartOfFilename.split('.')[1] + # NOTE: these files don't need to have cryptographically + # secure names + randStr = str(randint(10000000000000, 99999999999999)) # nosec personJson['icon']['url'] = \ - personId + '/avatar' + str(randint(10000000000000, 99999999999999)) + \ - '.' + existingExtension + personId + '/avatar' + randStr + '.' + existingExtension lastPartOfFilename = personJson['image']['url'].split('/')[-1] existingExtension = lastPartOfFilename.split('.')[1] + randStr = str(randint(10000000000000, 99999999999999)) # nosec personJson['image']['url'] = \ - personId + '/image' + str(randint(10000000000000, 99999999999999)) + \ - '.' + existingExtension + personId + '/image' + randStr + '.' + existingExtension def createPersonBase(baseDir: str, nickname: str, domain: str, port: int, @@ -197,13 +199,16 @@ def createPersonBase(baseDir: str, nickname: str, domain: str, port: int, approveFollowers = True personType = 'Application' + # NOTE: these image files don't need to have + # cryptographically secure names + imageUrl = \ personId + '/image' + \ - str(randint(10000000000000, 99999999999999)) + '.png' + str(randint(10000000000000, 99999999999999)) + '.png' # nosec iconUrl = \ personId + '/avatar' + \ - str(randint(10000000000000, 99999999999999)) + '.png' + str(randint(10000000000000, 99999999999999)) + '.png' # nosec contextDict = { 'Emoji': 'toot:Emoji', diff --git a/pgp.py b/pgp.py index 7f24cc546..f8dad7261 100644 --- a/pgp.py +++ b/pgp.py @@ -53,9 +53,39 @@ def getPGPpubKey(actorJson: {}) -> str: return '' +def getPGPfingerprint(actorJson: {}) -> str: + """Returns PGP fingerprint for the given actor + """ + if not actorJson.get('attachment'): + return '' + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue['name'].lower().startswith('openpgp'): + continue + if not propertyValue.get('type'): + continue + if not propertyValue.get('value'): + continue + if propertyValue['type'] != 'PropertyValue': + continue + if len(propertyValue['value']) < 10: + continue + return propertyValue['value'] + return '' + + def setEmailAddress(actorJson: {}, emailAddress: str) -> None: """Sets the email address for the given actor """ + notEmailAddress = False + if '@' not in emailAddress: + notEmailAddress = True + if '.' not in emailAddress: + notEmailAddress = True + if emailAddress.startswith('@'): + notEmailAddress = True + if not actorJson.get('attachment'): actorJson['attachment'] = [] @@ -72,12 +102,7 @@ def setEmailAddress(actorJson: {}, emailAddress: str) -> None: break if propertyFound: actorJson['attachment'].remove(propertyFound) - - if '@' not in emailAddress: - return - if '.' not in emailAddress: - return - if emailAddress.startswith('@'): + if notEmailAddress: return for propertyValue in actorJson['attachment']: @@ -103,6 +128,13 @@ def setEmailAddress(actorJson: {}, emailAddress: str) -> None: def setPGPpubKey(actorJson: {}, PGPpubKey: str) -> None: """Sets a PGP public key for the given actor """ + removeKey = False + if not PGPpubKey: + removeKey = True + else: + if '--BEGIN PGP PUBLIC KEY' not in PGPpubKey: + removeKey = True + if not actorJson.get('attachment'): actorJson['attachment'] = [] @@ -119,8 +151,7 @@ def setPGPpubKey(actorJson: {}, PGPpubKey: str) -> None: break if propertyFound: actorJson['attachment'].remove(propertyValue) - - if '--BEGIN PGP PUBLIC KEY' not in PGPpubKey: + if removeKey: return for propertyValue in actorJson['attachment']: @@ -141,3 +172,52 @@ def setPGPpubKey(actorJson: {}, PGPpubKey: str) -> None: "value": PGPpubKey } actorJson['attachment'].append(newPGPpubKey) + + +def setPGPfingerprint(actorJson: {}, fingerprint: str) -> None: + """Sets a PGP fingerprint for the given actor + """ + removeFingerprint = False + if not fingerprint: + removeFingerprint = True + else: + if len(fingerprint) < 10: + removeFingerprint = True + + if not actorJson.get('attachment'): + actorJson['attachment'] = [] + + # remove any existing value + propertyFound = None + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue.get('type'): + continue + if not propertyValue['name'].lower().startswith('openpgp'): + continue + propertyFound = propertyValue + break + if propertyFound: + actorJson['attachment'].remove(propertyValue) + if removeFingerprint: + return + + for propertyValue in actorJson['attachment']: + if not propertyValue.get('name'): + continue + if not propertyValue.get('type'): + continue + if not propertyValue['name'].lower().startswith('openpgp'): + continue + if propertyValue['type'] != 'PropertyValue': + continue + propertyValue['value'] = fingerprint.strip() + return + + newPGPfingerprint = { + "name": "OpenPGP", + "type": "PropertyValue", + "value": fingerprint + } + actorJson['attachment'].append(newPGPfingerprint) diff --git a/posts.py b/posts.py index 31319e828..8dd550806 100644 --- a/posts.py +++ b/posts.py @@ -146,11 +146,14 @@ def getUserUrl(wfRequest: {}) -> str: def parseUserFeed(session, feedUrl: str, asHeader: {}, projectVersion: str, httpPrefix: str, - domain: str) -> None: + domain: str, depth=0) -> {}: + if depth > 10: + return None + feedJson = getJson(session, feedUrl, asHeader, None, projectVersion, httpPrefix, domain) if not feedJson: - return + return None if 'orderedItems' in feedJson: for item in feedJson['orderedItems']: @@ -168,9 +171,10 @@ def parseUserFeed(session, feedUrl: str, asHeader: {}, userFeed = \ parseUserFeed(session, nextUrl, asHeader, projectVersion, httpPrefix, - domain) - for item in userFeed: - yield item + domain, depth+1) + if userFeed: + for item in userFeed: + yield item elif isinstance(nextUrl, dict): userFeed = nextUrl if userFeed.get('orderedItems'): @@ -440,6 +444,58 @@ def getPosts(session, outboxUrl: str, maxPosts: int, return personPosts +def getPostDomains(session, outboxUrl: str, maxPosts: int, + maxMentions: int, + maxEmoji: int, maxAttachments: int, + federationList: [], + personCache: {}, + debug: bool, + projectVersion: str, httpPrefix: str, + domain: str, domainList=[]) -> []: + """Returns a list of domains referenced within public posts + """ + if not outboxUrl: + return [] + profileStr = 'https://www.w3.org/ns/activitystreams' + asHeader = { + 'Accept': 'application/activity+json; profile="' + profileStr + '"' + } + if '/outbox/' in outboxUrl: + asHeader = { + 'Accept': 'application/ld+json; profile="' + profileStr + '"' + } + + postDomains = domainList + + i = 0 + userFeed = parseUserFeed(session, outboxUrl, asHeader, + projectVersion, httpPrefix, domain) + for item in userFeed: + i += 1 + if i > maxPosts: + break + if not item.get('object'): + continue + if not isinstance(item['object'], dict): + continue + if item['object'].get('inReplyTo'): + postDomain, postPort = \ + getDomainFromActor(item['object']['inReplyTo']) + if postDomain not in postDomains: + postDomains.append(postDomain) + + if item['object'].get('tag'): + for tagItem in item['object']['tag']: + tagType = tagItem['type'].lower() + if tagType == 'mention': + if tagItem.get('href'): + postDomain, postPort = \ + getDomainFromActor(tagItem['href']) + if postDomain not in postDomains: + postDomains.append(postDomain) + return postDomains + + def deleteAllPosts(baseDir: str, nickname: str, domain: str, boxname: str) -> None: """Deletes all posts for a person from inbox or outbox @@ -2933,6 +2989,54 @@ def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str, projectVersion, httpPrefix, domain) +def getPublicPostDomains(baseDir: str, nickname: str, domain: str, + proxyType: str, port: int, httpPrefix: str, + debug: bool, projectVersion: str, + domainList=[]) -> []: + """ Returns a list of domains referenced within public posts + """ + session = createSession(proxyType) + if not session: + return domainList + personCache = {} + cachedWebfingers = {} + federationList = [] + + domainFull = domain + if port: + if port != 80 and port != 443: + if ':' not in domain: + domainFull = domain + ':' + str(port) + handle = httpPrefix + "://" + domainFull + "/@" + nickname + wfRequest = \ + webfingerHandle(session, handle, httpPrefix, cachedWebfingers, + domain, projectVersion) + if not wfRequest: + return domainList + if not isinstance(wfRequest, dict): + print('Webfinger for ' + handle + ' did not return a dict. ' + + str(wfRequest)) + return domainList + + (personUrl, pubKeyId, pubKey, + personId, shaedInbox, + capabilityAcquisition, + avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest, + personCache, + projectVersion, httpPrefix, + nickname, domain, 'outbox') + maxMentions = 99 + maxEmoji = 99 + maxAttachments = 5 + postDomains = \ + getPostDomains(session, personUrl, 64, maxMentions, maxEmoji, + maxAttachments, federationList, + personCache, debug, + projectVersion, httpPrefix, domain, domainList) + postDomains.sort() + return postDomains + + def sendCapabilitiesUpdate(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, port: int, followerUrl, updateCaps: [], diff --git a/security_audit b/security_audit new file mode 100755 index 000000000..5a47f8c46 --- /dev/null +++ b/security_audit @@ -0,0 +1,2 @@ +#!/bin/bash +bandit *.py -x tests.py \ No newline at end of file diff --git a/socnet.py b/socnet.py new file mode 100644 index 000000000..fca13dc26 --- /dev/null +++ b/socnet.py @@ -0,0 +1,83 @@ +__filename__ = "socnet.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.1.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +from session import createSession +from webfinger import webfingerHandle +from posts import getPersonBox +from posts import getPostDomains + + +def instancesGraph(baseDir: str, handles: str, + proxyType: str, + port: int, httpPrefix: str, + debug: bool, projectVersion: str) -> str: + """ Returns a dot graph of federating instances + based upon a few sample handles. + The handles argument should contain a comma separated list + of handles on different instances + """ + dotGraphStr = 'digraph instances {\n' + if ',' not in handles: + return dotGraphStr + '}\n' + session = createSession(proxyType) + if not session: + return dotGraphStr + '}\n' + + personCache = {} + cachedWebfingers = {} + federationList = [] + maxMentions = 99 + maxEmoji = 99 + maxAttachments = 5 + + personHandles = handles.split(',') + for handle in personHandles: + handle = handle.strip() + if handle.startswith('@'): + handle = handle[1:] + if '@' not in handle: + continue + + nickname = handle.split('@')[0] + domain = handle.split('@')[1] + + domainFull = domain + if port: + if port != 80 and port != 443: + if ':' not in domain: + domainFull = domain + ':' + str(port) + handle = httpPrefix + "://" + domainFull + "/@" + nickname + wfRequest = \ + webfingerHandle(session, handle, httpPrefix, + cachedWebfingers, + domain, projectVersion) + if not wfRequest: + return dotGraphStr + '}\n' + if not isinstance(wfRequest, dict): + print('Webfinger for ' + handle + ' did not return a dict. ' + + str(wfRequest)) + return dotGraphStr + '}\n' + + (personUrl, pubKeyId, pubKey, + personId, shaedInbox, + capabilityAcquisition, + avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest, + personCache, + projectVersion, httpPrefix, + nickname, domain, 'outbox') + postDomains = \ + getPostDomains(session, personUrl, 64, maxMentions, maxEmoji, + maxAttachments, federationList, + personCache, debug, + projectVersion, httpPrefix, domain, []) + postDomains.sort() + for fedDomain in postDomains: + dotLineStr = ' "' + domain + '" -> "' + fedDomain + '";\n' + if dotLineStr not in dotGraphStr: + dotGraphStr += dotLineStr + return dotGraphStr + '}\n' diff --git a/ssb.py b/ssb.py index 0f393044c..93f2e4935 100644 --- a/ssb.py +++ b/ssb.py @@ -41,6 +41,18 @@ def getSSBAddress(actorJson: {}) -> str: def setSSBAddress(actorJson: {}, ssbAddress: str) -> None: """Sets an ssb address for the given actor """ + notSSBAddress = False + if not ssbAddress.startswith('@'): + notSSBAddress = True + if '=.' not in ssbAddress: + notSSBAddress = True + if '"' in ssbAddress: + notSSBAddress = True + if ' ' in ssbAddress: + notSSBAddress = True + if ',' in ssbAddress: + notSSBAddress = True + if not actorJson.get('attachment'): actorJson['attachment'] = [] @@ -57,16 +69,7 @@ def setSSBAddress(actorJson: {}, ssbAddress: str) -> None: break if propertyFound: actorJson['attachment'].remove(propertyFound) - - if not ssbAddress.startswith('@'): - return - if '=.' not in ssbAddress: - return - if '"' in ssbAddress: - return - if ' ' in ssbAddress: - return - if ',' in ssbAddress: + if notSSBAddress: return for propertyValue in actorJson['attachment']: diff --git a/tests.py b/tests.py index a0a60dfd0..12faa4634 100644 --- a/tests.py +++ b/tests.py @@ -64,6 +64,8 @@ from media import getAttachmentMediaType from delete import sendDeleteViaServer from inbox import validInbox from inbox import validInboxFilenames +from content import dangerousMarkup +from content import removeHtml from content import addWebLinks from content import replaceEmojiFromTags from content import addHtmlTags @@ -1873,8 +1875,49 @@ def testSiteIsActive(): assert(not siteIsActive('https://notarealwebsite.a.b.c')) +def testRemoveHtml(): + print('testRemoveHtml') + testStr = 'This string has no html.' + assert(removeHtml(testStr) == testStr) + testStr = 'This string has html.' + assert(removeHtml(testStr) == 'This string has html.') + + +def testDangerousMarkup(): + print('testDangerousMarkup') + content = '

This is a valid message

' + assert(not dangerousMarkup(content)) + content = 'This is a valid message without markup' + assert(not dangerousMarkup(content)) + content = '

This is a valid-looking message. But wait... ' + \ + '

' + assert(dangerousMarkup(content)) + content = '

This is a valid-looking message. But wait... ' + \ + '