diff --git a/Dockerfile b/Dockerfile index aff6cc08f..17ded0a72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,6 @@ RUN apt-get update && \ python3-idna \ libimage-exiftool-perl \ python3-flake8 \ - python3-pyld \ python3-django-timezone-field \ tor RUN adduser --system --home=/opt/epicyon --group epicyon diff --git a/README.md b/README.md index d06ed35ed..ab8f6494d 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ Add issues on https://gitlab.com/bashrc2/epicyon/-/issues
Epicyon, meaning "more than a dog". Largest of the Borophaginae which lived in North America 20-5 million years ago.
- + - + -Epicyon is a modern [ActivityPub](https://www.w3.org/TR/activitypub) compliant server implementing both S2S and C2S protocols and sutable for installation on single board computers. It includes features such as moderation tools, post expiry, content warnings, image descriptions and perimeter defense against adversaries. It contains *no javascript* and uses HTML+CSS with a Python backend. +Epicyon is a modern [ActivityPub](https://www.w3.org/TR/activitypub) compliant server implementing both S2S and C2S protocols and sutable for installation on single board computers. It includes features such as moderation tools, post expiry, content warnings, image descriptions, news feed and perimeter defense against adversaries. It contains *no javascript* and uses HTML+CSS with a Python backend. [Project Goals](README_goals.md) - [Commandline interface](README_commandline.md) - [Customizations](README_customizations.md) - [Code of Conduct](code-of-conduct.md) @@ -16,7 +16,9 @@ Matrix room: **#epicyon:matrix.freedombone.net** Includes emojis designed by [OpenMoji](https://openmoji.org) – the open-source emoji and icon project. License: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0). Blob Cat Emoji and Meowmoji were made by Nitro Blob Hub, licensed under [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0). - + + + ## Package Dependencies @@ -29,7 +31,7 @@ sudo pacman -S tor python-pip python-pysocks python-pycryptodome \ imagemagick python-pillow python-requests \ perl-image-exiftool python-numpy python-dateutil \ certbot flake8 bandit -sudo pip3 install pyLD pyqrcode pypng +sudo pip3 install pyqrcode pypng ``` Or on Debian: @@ -41,7 +43,7 @@ sudo apt install -y \ python3-crypto python3-pycryptodome \ python3-dateutil python3-pil.imagetk python3-idna python3-requests \ - python3-pyld python3-django-timezone-field \ + python3-django-timezone-field \ libimage-exiftool-perl python3-flake8 \ python3-pyqrcode python3-png python3-bandit \ certbot nginx diff --git a/config.py b/config.py deleted file mode 100644 index b290797bd..000000000 --- a/config.py +++ /dev/null @@ -1,46 +0,0 @@ -__filename__ = "config.py" -__author__ = "Bob Mottram" -__license__ = "AGPL3+" -__version__ = "1.1.0" -__maintainer__ = "Bob Mottram" -__email__ = "bob@freedombone.net" -__status__ = "Production" - -import os -from utils import loadJson -from utils import saveJson - - -def createConfig(baseDir: str) -> None: - """Creates a configuration file - """ - configFilename = baseDir + '/config.json' - if os.path.isfile(configFilename): - return - configJson = { - } - saveJson(configJson, configFilename) - - -def setConfigParam(baseDir: str, variableName: str, variableValue) -> None: - """Sets a configuration value - """ - createConfig(baseDir) - configFilename = baseDir + '/config.json' - configJson = {} - if os.path.isfile(configFilename): - configJson = loadJson(configFilename) - configJson[variableName] = variableValue - saveJson(configJson, configFilename) - - -def getConfigParam(baseDir: str, variableName: str): - """Gets a configuration value - """ - createConfig(baseDir) - configFilename = baseDir + '/config.json' - configJson = loadJson(configFilename) - if configJson: - if configJson.get(variableName): - return configJson[variableName] - return None diff --git a/content.py b/content.py index d1a170192..8b140c1f4 100644 --- a/content.py +++ b/content.py @@ -14,6 +14,23 @@ from utils import fileLastModified from utils import getLinkPrefixes +def removeHtmlTag(htmlStr: str, tag: str) -> str: + """Removes a given tag from a html string + """ + tagFound = True + while tagFound: + matchStr = ' ' + tag + '="' + if matchStr not in htmlStr: + tagFound = False + break + sections = htmlStr.split(matchStr, 1) + if '"' not in sections[1]: + tagFound = False + break + htmlStr = sections[0] + sections[1].split('"', 1)[1] + return htmlStr + + def removeQuotesWithinQuotes(content: str) -> str: """Removes any blockquote inside blockquote """ @@ -247,8 +264,10 @@ def addMusicTag(content: str, tag: str) -> str: """If a music link is found then ensure that the post is tagged appropriately """ + if '#podcast' in content or '#documentary' in content: + return content if '#' not in tag: - tag = '#'+tag + tag = '#' + tag if tag in content: return content musicSites = ('soundcloud.com', 'bandcamp.com') diff --git a/daemon.py b/daemon.py index 2b242ed2b..c5a815729 100644 --- a/daemon.py +++ b/daemon.py @@ -12,6 +12,7 @@ import json import time import locale import urllib.parse +import datetime from socket import error as SocketError import errno from functools import partial @@ -54,7 +55,7 @@ from person import registerAccount from person import personLookup from person import personBoxJson from person import createSharedInbox -from person import isSuspended +from person import createNewsInbox from person import suspendAccount from person import unsuspendAccount from person import removeAccount @@ -62,6 +63,7 @@ from person import canRemovePost from person import personSnooze from person import personUnsnooze from posts import isModerator +from posts import isEditor from posts import mutePost from posts import unmutePost from posts import createQuestionPost @@ -101,10 +103,9 @@ from blocking import removeGlobalBlock from blocking import isBlockedHashtag from blocking import isBlockedDomain from blocking import getDomainBlocklist -from config import setConfigParam -from config import getConfigParam from roles import setRole from roles import clearModeratorStatus +from roles import clearEditorStatus from blog import htmlBlogPageRSS2 from blog import htmlBlogPageRSS3 from blog import htmlBlogView @@ -122,6 +123,7 @@ from webinterface import htmlInboxDMs from webinterface import htmlInboxReplies from webinterface import htmlInboxMedia from webinterface import htmlInboxBlogs +from webinterface import htmlInboxNews from webinterface import htmlUnblockConfirm from webinterface import htmlPersonOptions from webinterface import htmlIndividualPost @@ -140,6 +142,8 @@ from webinterface import htmlNewPost from webinterface import htmlFollowConfirm from webinterface import htmlCalendar from webinterface import htmlSearch +from webinterface import htmlNewswireMobile +from webinterface import htmlLinksMobile from webinterface import htmlSearchEmoji from webinterface import htmlSearchEmojiTextEntry from webinterface import htmlUnfollowConfirm @@ -147,6 +151,7 @@ from webinterface import htmlProfileAfterSearch from webinterface import htmlEditProfile from webinterface import htmlEditLinks from webinterface import htmlEditNewswire +from webinterface import htmlEditNewsPost from webinterface import htmlTermsOfService from webinterface import htmlSkillsSearch from webinterface import htmlHistorySearch @@ -159,6 +164,8 @@ from shares import getSharesFeedForPerson from shares import addShare from shares import removeShare from shares import expireShares +from utils import setConfigParam +from utils import getConfigParam from utils import removeIdEnding from utils import updateLikesCollection from utils import undoLikesCollectionEntry @@ -174,6 +181,7 @@ from utils import getStatusNumber from utils import urlPermitted from utils import loadJson from utils import saveJson +from utils import isSuspended from manualapprove import manualDenyFollowRequest from manualapprove import manualApproveFollowRequest from announce import createAnnounce @@ -204,8 +212,8 @@ from devices import E2EEdevicesCollection from devices import E2EEvalidDevice from devices import E2EEaddDevice from newswire import getRSSfromDict -from newswire import runNewswireWatchdog -from newswire import runNewswireDaemon +from newsdaemon import runNewswireWatchdog +from newsdaemon import runNewswireDaemon import os @@ -218,6 +226,8 @@ maxPostsInMediaFeed = 6 # Blogs can be longer, so don't show many per page maxPostsInBlogsFeed = 4 +maxPostsInNewsFeed = 10 + # Maximum number of entries in returned rss.xml maxPostsInRSSFeed = 10 @@ -730,11 +740,12 @@ class PubServer(BaseHTTPRequestHandler): return False if self.server.debug: print('DEBUG: mastodon api ' + self.path) - if self.path == '/api/v1/instance': - adminNickname = getConfigParam(self.server.baseDir, 'admin') + adminNickname = getConfigParam(self.server.baseDir, 'admin') + if adminNickname and self.path == '/api/v1/instance': instanceDescriptionShort = \ getConfigParam(self.server.baseDir, 'instanceDescriptionShort') + instanceDescriptionShort = 'Yet another Epicyon Instance' instanceDescription = getConfigParam(self.server.baseDir, 'instanceDescription') instanceTitle = getConfigParam(self.server.baseDir, @@ -946,7 +957,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.allowDeletion, self.server.proxyType, version, self.server.debug, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) def _postToOutboxThread(self, messageJson: {}) -> bool: """Creates a thread to send a post @@ -1091,7 +1103,7 @@ class PubServer(BaseHTTPRequestHandler): return True elif '/' + nickname + '?' in self.path: return True - elif self.path.endswith('/'+nickname): + elif self.path.endswith('/' + nickname): return True print('AUTH: nickname ' + nickname + ' was not found in path ' + self.path) @@ -1228,6 +1240,11 @@ class PubServer(BaseHTTPRequestHandler): loginNickname, loginPassword, register = \ htmlGetLoginCredentials(loginParams, self.server.lastLoginTime) if loginNickname: + if loginNickname == 'news' or loginNickname == 'inbox': + print('Invalid username login: ' + loginNickname) + self._clearLoginDetails(loginNickname, callingDomain) + self.server.POSTbusy = False + return self.server.lastLoginTime = int(time.time()) if register: if not registerAccount(baseDir, httpPrefix, domain, port, @@ -2240,7 +2257,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.personCache, httpPrefix, self.server.projectVersion, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) if hashtagStr: msg = hashtagStr.encode('utf-8') self._login_headers('text/html', @@ -2285,7 +2303,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers, self.server.personCache, port, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) if historyStr: msg = historyStr.encode('utf-8') self._login_headers('text/html', @@ -2328,7 +2347,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.personCache, self.server.debug, self.server.projectVersion, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) if profileStr: msg = profileStr.encode('utf-8') self._login_headers('text/html', @@ -2714,10 +2734,10 @@ class PubServer(BaseHTTPRequestHandler): # get the nickname nickname = getNicknameFromActor(actorStr) - moderator = None + editor = None if nickname: - moderator = isModerator(baseDir, nickname) - if not nickname or not moderator: + editor = isEditor(baseDir, nickname) + if not nickname or not editor: if callingDomain.endswith('.onion') and \ onionDomain: actorStr = \ @@ -2919,6 +2939,151 @@ class PubServer(BaseHTTPRequestHandler): cookie, callingDomain) self.server.POSTbusy = False + def _newsPostEdit(self, callingDomain: str, cookie: str, + authorized: bool, path: str, + baseDir: str, httpPrefix: str, + domain: str, domainFull: str, + onionDomain: str, i2pDomain: str, debug: bool, + defaultTimeline: str): + """edits a news post + """ + usersPath = path.replace('/newseditdata', '') + usersPath = usersPath.replace('/editnewspost', '') + actorStr = httpPrefix + '://' + domainFull + usersPath + if ' boundary=' in self.headers['Content-type']: + boundary = self.headers['Content-type'].split('boundary=')[1] + if ';' in boundary: + boundary = boundary.split(';')[0] + + # get the nickname + nickname = getNicknameFromActor(actorStr) + editorRole = None + if nickname: + editorRole = isEditor(baseDir, nickname) + if not nickname or not editorRole: + if callingDomain.endswith('.onion') and \ + onionDomain: + actorStr = \ + 'http://' + onionDomain + usersPath + elif (callingDomain.endswith('.i2p') and + i2pDomain): + actorStr = \ + 'http://' + i2pDomain + usersPath + if not nickname: + print('WARN: nickname not found in ' + actorStr) + else: + print('WARN: nickname is not an editor' + actorStr) + self._redirect_headers(actorStr + '/tlnews', + cookie, callingDomain) + self.server.POSTbusy = False + return + + length = int(self.headers['Content-length']) + + # check that the POST isn't too large + if length > self.server.maxPostLength: + if callingDomain.endswith('.onion') and \ + onionDomain: + actorStr = \ + 'http://' + onionDomain + usersPath + elif (callingDomain.endswith('.i2p') and + i2pDomain): + actorStr = \ + 'http://' + i2pDomain + usersPath + print('Maximum news data length exceeded ' + str(length)) + self._redirect_headers(actorStr + 'tlnews', + cookie, callingDomain) + self.server.POSTbusy = False + return + + try: + # read the bytes of the http form POST + postBytes = self.rfile.read(length) + except SocketError as e: + if e.errno == errno.ECONNRESET: + print('WARN: connection was reset while ' + + 'reading bytes from http form POST') + else: + print('WARN: error while reading bytes ' + + 'from http form POST') + self.send_response(400) + self.end_headers() + self.server.POSTbusy = False + return + except ValueError as e: + print('ERROR: failed to read bytes for POST') + print(e) + self.send_response(400) + self.end_headers() + self.server.POSTbusy = False + return + + # extract all of the text fields into a dict + fields = \ + extractTextFieldsInPOST(postBytes, boundary, debug) + newsPostUrl = None + newsPostTitle = None + newsPostContent = None + if fields.get('newsPostUrl'): + newsPostUrl = fields['newsPostUrl'] + if fields.get('newsPostTitle'): + newsPostTitle = fields['newsPostTitle'] + if fields.get('editedNewsPost'): + newsPostContent = fields['editedNewsPost'] + + if newsPostUrl and newsPostContent and newsPostTitle: + # load the post + postFilename = \ + locatePost(baseDir, nickname, domain, + newsPostUrl) + if postFilename: + postJsonObject = loadJson(postFilename) + # update the content and title + postJsonObject['object']['summary'] = \ + newsPostTitle + postJsonObject['object']['content'] = \ + newsPostContent + # remove the html from post cache + cachedPost = \ + baseDir + '/accounts/' + \ + nickname + '@' + domain + \ + '/postcache/' + newsPostUrl + '.html' + if os.path.isfile(cachedPost): + os.remove(cachedPost) + # update newswire + pubDate = postJsonObject['object']['published'] + publishedDate = \ + datetime.datetime.strptime(pubDate, + "%Y-%m-%dT%H:%M:%SZ") + if self.server.newswire.get(str(publishedDate)): + self.server.newswire[publishedDate][0] = \ + newsPostTitle + self.server.newswire[publishedDate][4] = \ + newsPostContent + # save newswire + newswireStateFilename = \ + baseDir + '/accounts/.newswirestate.json' + try: + saveJson(self.server.newswire, + newswireStateFilename) + except Exception as e: + print('ERROR saving newswire state, ' + str(e)) + # save the news post + saveJson(postJsonObject, postFilename) + + # redirect back to the default timeline + if callingDomain.endswith('.onion') and \ + onionDomain: + actorStr = \ + 'http://' + onionDomain + usersPath + elif (callingDomain.endswith('.i2p') and + i2pDomain): + actorStr = \ + 'http://' + i2pDomain + usersPath + self._redirect_headers(actorStr + '/tlnews', + cookie, callingDomain) + self.server.POSTbusy = False + def _profileUpdate(self, callingDomain: str, cookie: str, authorized: bool, path: str, baseDir: str, httpPrefix: str, @@ -3146,7 +3311,8 @@ class PubServer(BaseHTTPRequestHandler): actorChanged = True if fields.get('themeDropdown'): setTheme(baseDir, - fields['themeDropdown']) + fields['themeDropdown'], + domain) # change email address currentEmailAddress = getEmailAddress(actorJson) @@ -3258,11 +3424,9 @@ class PubServer(BaseHTTPRequestHandler): # change instance title if fields.get('instanceTitle'): currInstanceTitle = \ - getConfigParam(baseDir, - 'instanceTitle') + getConfigParam(baseDir, 'instanceTitle') if fields['instanceTitle'] != currInstanceTitle: - setConfigParam(baseDir, - 'instanceTitle', + setConfigParam(baseDir, 'instanceTitle', fields['instanceTitle']) # change YouTube alternate domain @@ -3301,8 +3465,7 @@ class PubServer(BaseHTTPRequestHandler): setConfigParam(baseDir, 'instanceDescriptionShort', '') currInstanceDescription = \ - getConfigParam(baseDir, - 'instanceDescription') + getConfigParam(baseDir, 'instanceDescription') if fields.get('instanceDescription'): if fields['instanceDescription'] != \ currInstanceDescription: @@ -3338,60 +3501,121 @@ class PubServer(BaseHTTPRequestHandler): if fields.get('moderators'): adminNickname = \ getConfigParam(baseDir, 'admin') - if path.startswith('/users/' + - adminNickname + '/'): - moderatorsFile = \ - baseDir + \ - '/accounts/moderators.txt' - clearModeratorStatus(baseDir) - if ',' in fields['moderators']: - # if the list was given as comma separated - modFile = open(moderatorsFile, "w+") - mods = fields['moderators'].split(',') - for modNick in mods: - modNick = modNick.strip() - modDir = baseDir + \ - '/accounts/' + modNick + \ - '@' + domain - if os.path.isdir(modDir): - modFile.write(modNick + '\n') - modFile.close() - mods = fields['moderators'].split(',') - for modNick in mods: - modNick = modNick.strip() - modDir = baseDir + \ - '/accounts/' + modNick + \ - '@' + domain - if os.path.isdir(modDir): - setRole(baseDir, - modNick, domain, - 'instance', 'moderator') - else: - # nicknames on separate lines - modFile = open(moderatorsFile, "w+") - mods = fields['moderators'].split('\n') - for modNick in mods: - modNick = modNick.strip() - modDir = \ - baseDir + \ - '/accounts/' + modNick + \ - '@' + domain - if os.path.isdir(modDir): - modFile.write(modNick + '\n') - modFile.close() - mods = fields['moderators'].split('\n') - for modNick in mods: - modNick = modNick.strip() - modDir = \ - baseDir + \ - '/accounts/' + \ - modNick + '@' + \ - domain - if os.path.isdir(modDir): - setRole(baseDir, - modNick, domain, - 'instance', - 'moderator') + if adminNickname: + if path.startswith('/users/' + + adminNickname + '/'): + moderatorsFile = \ + baseDir + \ + '/accounts/moderators.txt' + clearModeratorStatus(baseDir) + if ',' in fields['moderators']: + # if the list was given as comma separated + modFile = open(moderatorsFile, "w+") + mods = fields['moderators'].split(',') + for modNick in mods: + modNick = modNick.strip() + modDir = baseDir + \ + '/accounts/' + modNick + \ + '@' + domain + if os.path.isdir(modDir): + modFile.write(modNick + '\n') + modFile.close() + mods = fields['moderators'].split(',') + for modNick in mods: + modNick = modNick.strip() + modDir = baseDir + \ + '/accounts/' + modNick + \ + '@' + domain + if os.path.isdir(modDir): + setRole(baseDir, + modNick, domain, + 'instance', 'moderator') + else: + # nicknames on separate lines + modFile = open(moderatorsFile, "w+") + mods = fields['moderators'].split('\n') + for modNick in mods: + modNick = modNick.strip() + modDir = \ + baseDir + \ + '/accounts/' + modNick + \ + '@' + domain + if os.path.isdir(modDir): + modFile.write(modNick + '\n') + modFile.close() + mods = fields['moderators'].split('\n') + for modNick in mods: + modNick = modNick.strip() + modDir = \ + baseDir + \ + '/accounts/' + \ + modNick + '@' + \ + domain + if os.path.isdir(modDir): + setRole(baseDir, + modNick, domain, + 'instance', + 'moderator') + + # change site editors list + if fields.get('editors'): + adminNickname = \ + getConfigParam(baseDir, 'admin') + if adminNickname: + if path.startswith('/users/' + + adminNickname + '/'): + editorsFile = \ + baseDir + \ + '/accounts/editors.txt' + clearEditorStatus(baseDir) + if ',' in fields['editors']: + # if the list was given as comma separated + edFile = open(editorsFile, "w+") + eds = fields['editors'].split(',') + for edNick in eds: + edNick = edNick.strip() + edDir = baseDir + \ + '/accounts/' + edNick + \ + '@' + domain + if os.path.isdir(edDir): + edFile.write(edNick + '\n') + edFile.close() + eds = fields['editors'].split(',') + for edNick in eds: + edNick = edNick.strip() + edDir = baseDir + \ + '/accounts/' + edNick + \ + '@' + domain + if os.path.isdir(edDir): + setRole(baseDir, + edNick, domain, + 'instance', 'editor') + else: + # nicknames on separate lines + edFile = open(editorsFile, "w+") + eds = fields['editors'].split('\n') + for edNick in eds: + edNick = edNick.strip() + edDir = \ + baseDir + \ + '/accounts/' + edNick + \ + '@' + domain + if os.path.isdir(edDir): + edFile.write(edNick + '\n') + edFile.close() + eds = fields['editors'].split('\n') + for edNick in eds: + edNick = edNick.strip() + edDir = \ + baseDir + \ + '/accounts/' + \ + edNick + '@' + \ + domain + if os.path.isdir(edDir): + setRole(baseDir, + edNick, domain, + 'instance', + 'editor') # remove scheduled posts if fields.get('removeScheduledPosts'): @@ -3427,7 +3651,7 @@ class PubServer(BaseHTTPRequestHandler): '.etag') currTheme = getTheme(baseDir) if currTheme: - setTheme(baseDir, currTheme) + setTheme(baseDir, currTheme, domain) # change media instance status if fields.get('mediaInstance'): @@ -3436,6 +3660,7 @@ class PubServer(BaseHTTPRequestHandler): if fields['mediaInstance'] == 'on': self.server.mediaInstance = True self.server.blogsInstance = False + self.server.newsInstance = False self.server.defaultTimeline = 'tlmedia' setConfigParam(baseDir, "mediaInstance", @@ -3443,6 +3668,9 @@ class PubServer(BaseHTTPRequestHandler): setConfigParam(baseDir, "blogsInstance", self.server.blogsInstance) + setConfigParam(baseDir, + "newsInstance", + self.server.newsInstance) else: if self.server.mediaInstance: self.server.mediaInstance = False @@ -3451,6 +3679,32 @@ class PubServer(BaseHTTPRequestHandler): "mediaInstance", self.server.mediaInstance) + # change news instance status + if fields.get('newsInstance'): + self.server.newsInstance = False + self.server.defaultTimeline = 'inbox' + if fields['newsInstance'] == 'on': + self.server.newsInstance = True + self.server.blogsInstance = False + self.server.mediaInstance = False + self.server.defaultTimeline = 'tlnews' + setConfigParam(baseDir, + "mediaInstance", + self.server.mediaInstance) + setConfigParam(baseDir, + "blogsInstance", + self.server.blogsInstance) + setConfigParam(baseDir, + "newsInstance", + self.server.newsInstance) + else: + if self.server.newsInstance: + self.server.newsInstance = False + self.server.defaultTimeline = 'inbox' + setConfigParam(baseDir, + "newsInstance", + self.server.mediaInstance) + # change blog instance status if fields.get('blogsInstance'): self.server.blogsInstance = False @@ -3458,6 +3712,7 @@ class PubServer(BaseHTTPRequestHandler): if fields['blogsInstance'] == 'on': self.server.blogsInstance = True self.server.mediaInstance = False + self.server.newsInstance = False self.server.defaultTimeline = 'tlblogs' setConfigParam(baseDir, "blogsInstance", @@ -3465,6 +3720,9 @@ class PubServer(BaseHTTPRequestHandler): setConfigParam(baseDir, "mediaInstance", self.server.mediaInstance) + setConfigParam(baseDir, + "newsInstance", + self.server.newsInstance) else: if self.server.blogsInstance: self.server.blogsInstance = False @@ -4347,7 +4605,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.personCache, httpPrefix, self.server.projectVersion, - self.server.YTReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) if hashtagStr: msg = hashtagStr.encode('utf-8') self._set_headers('text/html', len(msg), @@ -4657,6 +4916,100 @@ class PubServer(BaseHTTPRequestHandler): 'follow approve shown') self.server.GETbusy = False + def _newswireVote(self, callingDomain: str, path: str, + cookie: str, + baseDir: str, httpPrefix: str, + domain: str, domainFull: str, port: int, + onionDomain: str, i2pDomain: str, + GETstartTime, GETtimings: {}, + proxyType: str, debug: bool, + newswire: {}): + """Vote for a newswire item + """ + originPathStr = path.split('/newswirevote=')[0] + dateStr = \ + path.split('/newswirevote=')[1].replace('T', ' ') + '+00:00' + nickname = originPathStr.split('/users/')[1] + if '/' in nickname: + nickname = nickname.split('/')[0] + if newswire.get(dateStr): + if isModerator(baseDir, nickname): + if 'vote:' + nickname not in newswire[dateStr][2]: + newswire[dateStr][2].append('vote:' + nickname) + filename = newswire[dateStr][3] + newswireStateFilename = \ + baseDir + '/accounts/.newswirestate.json' + try: + saveJson(newswire, newswireStateFilename) + except Exception as e: + print('ERROR saving newswire state, ' + str(e)) + if filename: + saveJson(newswire[dateStr][2], + filename + '.votes') + + originPathStrAbsolute = \ + httpPrefix + '://' + domainFull + originPathStr + '/' + \ + self.server.defaultTimeline + if callingDomain.endswith('.onion') and onionDomain: + originPathStrAbsolute = \ + 'http://' + onionDomain + originPathStr + elif (callingDomain.endswith('.i2p') and i2pDomain): + originPathStrAbsolute = \ + 'http://' + i2pDomain + originPathStr + self._redirect_headers(originPathStrAbsolute, + cookie, callingDomain) + self._benchmarkGETtimings(GETstartTime, GETtimings, + 'unannounce done', + 'vote for newswite item') + self.server.GETbusy = False + + def _newswireUnvote(self, callingDomain: str, path: str, + cookie: str, + baseDir: str, httpPrefix: str, + domain: str, domainFull: str, port: int, + onionDomain: str, i2pDomain: str, + GETstartTime, GETtimings: {}, + proxyType: str, debug: bool, + newswire: {}): + """Remove vote for a newswire item + """ + originPathStr = path.split('/newswireunvote=')[0] + dateStr = \ + path.split('/newswireunvote=')[1].replace('T', ' ') + '+00:00' + nickname = originPathStr.split('/users/')[1] + if '/' in nickname: + nickname = nickname.split('/')[0] + if newswire.get(dateStr): + if isModerator(baseDir, nickname): + if 'vote:' + nickname in newswire[dateStr][2]: + newswire[dateStr][2].remove('vote:' + nickname) + filename = newswire[dateStr][3] + newswireStateFilename = \ + baseDir + '/accounts/.newswirestate.json' + try: + saveJson(newswire, newswireStateFilename) + except Exception as e: + print('ERROR saving newswire state, ' + str(e)) + if filename: + saveJson(newswire[dateStr][2], + filename + '.votes') + + originPathStrAbsolute = \ + httpPrefix + '://' + domainFull + originPathStr + '/' + \ + self.server.defaultTimeline + if callingDomain.endswith('.onion') and onionDomain: + originPathStrAbsolute = \ + 'http://' + onionDomain + originPathStr + elif (callingDomain.endswith('.i2p') and i2pDomain): + originPathStrAbsolute = \ + 'http://' + i2pDomain + originPathStr + self._redirect_headers(originPathStrAbsolute, + cookie, callingDomain) + self._benchmarkGETtimings(GETstartTime, GETtimings, + 'unannounce done', + 'unvote for newswite item') + self.server.GETbusy = False + def _followDenyButton(self, callingDomain: str, path: str, cookie: str, baseDir: str, httpPrefix: str, @@ -5159,7 +5512,8 @@ class PubServer(BaseHTTPRequestHandler): deleteUrl, httpPrefix, __version__, self.server.cachedWebfingers, self.server.personCache, callingDomain, - self.server.TYReplacementDomain) + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly) if deleteStr: self._set_headers('text/html', len(deleteStr), cookie, callingDomain) @@ -5358,7 +5712,8 @@ class PubServer(BaseHTTPRequestHandler): repliesJson, httpPrefix, projectVersion, - ytDomain) + ytDomain, + self.server.showPublishedDateOnly) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -5438,7 +5793,8 @@ class PubServer(BaseHTTPRequestHandler): repliesJson, httpPrefix, projectVersion, - ytDomain) + ytDomain, + self.server.showPublishedDateOnly) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -5513,6 +5869,7 @@ class PubServer(BaseHTTPRequestHandler): cachedWebfingers, self.server.personCache, YTReplacementDomain, + self.server.showPublishedDateOnly, actorJson['roles'], None, None) msg = msg.encode('utf-8') @@ -5571,6 +5928,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers YTReplacementDomain = \ self.server.YTReplacementDomain + showPublishedDateOnly = \ + self.server.showPublishedDateOnly msg = \ htmlProfile(defaultTimeline, recentPostsCache, @@ -5583,6 +5942,7 @@ class PubServer(BaseHTTPRequestHandler): cachedWebfingers, self.server.personCache, YTReplacementDomain, + showPublishedDateOnly, actorJson['skills'], None, None) msg = msg.encode('utf-8') @@ -5683,6 +6043,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion ytDomain = \ self.server.YTReplacementDomain + showPublishedDateOnly = \ + self.server.showPublishedDateOnly msg = \ htmlIndividualPost(recentPostsCache, maxRecentPosts, @@ -5698,7 +6060,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, projectVersion, likedBy, - ytDomain) + ytDomain, + showPublishedDateOnly) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -5790,6 +6153,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion ytDomain = \ self.server.YTReplacementDomain + showPublishedDateOnly = \ + self.server.showPublishedDateOnly msg = \ htmlIndividualPost(recentPostsCache, maxRecentPosts, @@ -5806,7 +6171,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, projectVersion, likedBy, - ytDomain) + ytDomain, + showPublishedDateOnly) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -5865,7 +6231,10 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'inbox', - authorized) + authorized, + 0, + self.server.positiveVoting, + self.server.votingTimeMins) if inboxFeed: if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -5893,7 +6262,10 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInFeed, 'inbox', - authorized) + authorized, + 0, + self.server.positiveVoting, + self.server.votingTimeMins) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -5917,7 +6289,9 @@ class PubServer(BaseHTTPRequestHandler): projectVersion, self._isMinimal(nickname), YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, 'show status done', @@ -5980,7 +6354,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'dm', - authorized) + authorized, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if inboxDMFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -6004,7 +6380,10 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInFeed, 'dm', - authorized) + authorized, + 0, + self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxDMs(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6024,7 +6403,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6080,7 +6461,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'tlreplies', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxRepliesFeed: inboxRepliesFeed = [] if self._requestHTTP(): @@ -6105,7 +6488,9 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInFeed, 'tlreplies', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxReplies(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6125,7 +6510,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6181,7 +6568,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInMediaFeed, 'tlmedia', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxMediaFeed: inboxMediaFeed = [] if self._requestHTTP(): @@ -6206,7 +6595,9 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInMediaFeed, 'tlmedia', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxMedia(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6226,7 +6617,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6282,7 +6675,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInBlogsFeed, 'tlblogs', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if not inboxBlogsFeed: inboxBlogsFeed = [] if self._requestHTTP(): @@ -6307,7 +6702,9 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInBlogsFeed, 'tlblogs', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlInboxBlogs(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6327,7 +6724,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6363,6 +6762,121 @@ class PubServer(BaseHTTPRequestHandler): return True return False + def _showNewsTimeline(self, authorized: bool, + callingDomain: str, path: str, + baseDir: str, httpPrefix: str, + domain: str, domainFull: str, port: int, + onionDomain: str, i2pDomain: str, + GETstartTime, GETtimings: {}, + proxyType: str, cookie: str, + debug: str) -> bool: + """Shows the news timeline + """ + if '/users/' in path: + if authorized: + inboxNewsFeed = \ + personBoxJson(self.server.recentPostsCache, + self.server.session, + baseDir, + domain, + port, + path, + httpPrefix, + maxPostsInNewsFeed, 'tlnews', + True, + self.server.newswireVotesThreshold, + self.server.positiveVoting, + self.server.votingTimeMins) + if not inboxNewsFeed: + inboxNewsFeed = [] + if self._requestHTTP(): + nickname = path.replace('/users/', '') + nickname = nickname.replace('/tlnews', '') + pageNumber = 1 + if '?page=' in nickname: + pageNumber = nickname.split('?page=')[1] + nickname = nickname.split('?page=')[0] + if pageNumber.isdigit(): + pageNumber = int(pageNumber) + else: + pageNumber = 1 + if 'page=' not in path: + # if no page was specified then show the first + inboxNewsFeed = \ + personBoxJson(self.server.recentPostsCache, + self.server.session, + baseDir, + domain, + port, + path + '?page=1', + httpPrefix, + maxPostsInBlogsFeed, 'tlnews', + True, + self.server.newswireVotesThreshold, + self.server.positiveVoting, + self.server.votingTimeMins) + currNickname = path.split('/users/')[1] + if '/' in currNickname: + currNickname = currNickname.split('/')[0] + moderator = isModerator(baseDir, currNickname) + editor = isEditor(baseDir, currNickname) + msg = \ + htmlInboxNews(self.server.defaultTimeline, + self.server.recentPostsCache, + self.server.maxRecentPosts, + self.server.translate, + pageNumber, maxPostsInNewsFeed, + self.server.session, + baseDir, + self.server.cachedWebfingers, + self.server.personCache, + nickname, + domain, + port, + inboxNewsFeed, + self.server.allowDeletion, + httpPrefix, + self.server.projectVersion, + self._isMinimal(nickname), + self.server.YTReplacementDomain, + self.server.showPublishedDateOnly, + self.server.newswire, + moderator, editor, + self.server.positiveVoting) + msg = msg.encode('utf-8') + self._set_headers('text/html', len(msg), + cookie, callingDomain) + self._write(msg) + self._benchmarkGETtimings(GETstartTime, GETtimings, + 'show blogs 2 done', + 'show news 2') + else: + # don't need authenticated fetch here because there is + # already the authorization check + msg = json.dumps(inboxNewsFeed, + ensure_ascii=False) + msg = msg.encode('utf-8') + self._set_headers('application/json', + len(msg), + None, callingDomain) + self._write(msg) + self.server.GETbusy = False + return True + else: + if debug: + nickname = 'news' + print('DEBUG: ' + nickname + + ' was not authorized to access ' + path) + if path != '/tlnews': + # not the news inbox + if debug: + print('DEBUG: GET access to news is unauthorized') + self.send_response(405) + self.end_headers() + self.server.GETbusy = False + return True + return False + def _showSharesTimeline(self, authorized: bool, callingDomain: str, path: str, baseDir: str, httpPrefix: str, @@ -6403,7 +6917,9 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6442,7 +6958,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'tlbookmarks', - authorized) + authorized, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if bookmarksFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -6468,7 +6986,9 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlbookmarks', - authorized) + authorized, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlBookmarks(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6488,7 +7008,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6546,7 +7068,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'tlevents', - authorized) + authorized, + 0, self.server.positiveVoting, + self.server.votingTimeMins) print('eventsFeed: ' + str(eventsFeed)) if eventsFeed: if self._requestHTTP(): @@ -6572,7 +7096,9 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, maxPostsInFeed, 'tlevents', - authorized) + authorized, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlEvents(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6592,7 +7118,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6642,7 +7170,10 @@ class PubServer(BaseHTTPRequestHandler): port, path, httpPrefix, maxPostsInFeed, 'outbox', - authorized) + authorized, + self.server.newswireVotesThreshold, + self.server.positiveVoting, + self.server.votingTimeMins) if outboxFeed: if self._requestHTTP(): nickname = \ @@ -6666,7 +7197,10 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInFeed, 'outbox', - authorized) + authorized, + self.server.newswireVotesThreshold, + self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlOutbox(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6686,7 +7220,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self._isMinimal(nickname), self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6729,7 +7265,9 @@ class PubServer(BaseHTTPRequestHandler): path, httpPrefix, maxPostsInFeed, 'moderation', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) if moderationFeed: if self._requestHTTP(): nickname = path.replace('/users/', '') @@ -6753,7 +7291,9 @@ class PubServer(BaseHTTPRequestHandler): path + '?page=1', httpPrefix, maxPostsInFeed, 'moderation', - True) + True, + 0, self.server.positiveVoting, + self.server.votingTimeMins) msg = \ htmlModeration(self.server.defaultTimeline, self.server.recentPostsCache, @@ -6772,7 +7312,9 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self.server.YTReplacementDomain, - self.server.newswire) + self.server.showPublishedDateOnly, + self.server.newswire, + self.server.positiveVoting) msg = msg.encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -6862,6 +7404,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers, self.server.personCache, self.server.YTReplacementDomain, + self.server.showPublishedDateOnly, shares, pageNumber, sharesPerPage) msg = msg.encode('utf-8') @@ -6948,6 +7491,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers, self.server.personCache, self.server.YTReplacementDomain, + self.server.showPublishedDateOnly, following, pageNumber, followsPerPage).encode('utf-8') @@ -7034,6 +7578,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers, self.server.personCache, self.server.YTReplacementDomain, + self.server.showPublishedDateOnly, followers, pageNumber, followsPerPage).encode('utf-8') @@ -7095,6 +7640,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.cachedWebfingers, self.server.personCache, self.server.YTReplacementDomain, + self.server.showPublishedDateOnly, None, None).encode('utf-8') self._set_headers('text/html', len(msg), cookie, callingDomain) @@ -7676,6 +8222,34 @@ class PubServer(BaseHTTPRequestHandler): return True return False + def _editNewsPost(self, callingDomain: str, path: str, + translate: {}, baseDir: str, + httpPrefix: str, domain: str, port: int, + domainFull: str, + cookie: str) -> bool: + """Show the edit screen for a news post + """ + if '/users/' in path and '/editnewspost=' in path: + postId = path.split('/editnewspost=')[1] + if '?' in postId: + postId = postId.split('?')[0] + postUrl = httpPrefix + '://' + domainFull + \ + '/users/news/statuses/' + postId + path = path.split('/editnewspost=')[0] + msg = htmlEditNewsPost(translate, baseDir, + path, domain, port, + httpPrefix, + postUrl).encode('utf-8') + if msg: + self._set_headers('text/html', len(msg), + cookie, callingDomain) + self._write(msg) + else: + self._404() + self.server.GETbusy = False + return True + return False + def _editEvent(self, callingDomain: str, path: str, httpPrefix: str, domain: str, domainFull: str, baseDir: str, translate: {}, @@ -8510,6 +9084,48 @@ class PubServer(BaseHTTPRequestHandler): 'permitted directory', 'login shown done') + if authorized and htmlGET and '/users/' in self.path and \ + self.path.endswith('/newswiremobile'): + nickname = getNicknameFromActor(self.path) + if not nickname: + self._404() + self.server.GETbusy = False + return + timelinePath = \ + '/users/' + nickname + '/' + self.server.defaultTimeline + msg = htmlNewswireMobile(self.server.baseDir, + nickname, + self.server.domain, + self.server.domainFull, + self.server.httpPrefix, + self.server.translate, + self.server.newswire, + self.server.positiveVoting, + timelinePath).encode('utf-8') + self._set_headers('text/html', len(msg), cookie, callingDomain) + self._write(msg) + self.server.GETbusy = False + return + + if authorized and htmlGET and '/users/' in self.path and \ + self.path.endswith('/linksmobile'): + nickname = getNicknameFromActor(self.path) + if not nickname: + self._404() + self.server.GETbusy = False + return + timelinePath = \ + '/users/' + nickname + '/' + self.server.defaultTimeline + msg = htmlLinksMobile(self.server.baseDir, nickname, + self.server.domainFull, + self.server.httpPrefix, + self.server.translate, + timelinePath).encode('utf-8') + self._set_headers('text/html', len(msg), cookie, callingDomain) + self._write(msg) + self.server.GETbusy = False + return + # hashtag search if self.path.startswith('/tags/') or \ (authorized and '/tags/' in self.path): @@ -8550,13 +9166,16 @@ class PubServer(BaseHTTPRequestHandler): nickname = nickname.split('/')[0] self._setMinimal(nickname, not self._isMinimal(nickname)) if not (self.server.mediaInstance or - self.server.blogsInstance): + self.server.blogsInstance or + self.server.newsInstance): self.path = '/users/' + nickname + '/inbox' else: if self.server.blogsInstance: self.path = '/users/' + nickname + '/tlblogs' - else: + elif self.server.mediaInstance: self.path = '/users/' + nickname + '/tlmedia' + else: + self.path = '/users/' + nickname + '/tlnews' # search for a fediverse address, shared item or emoji # from the web interface by selecting search icon @@ -8688,6 +9307,42 @@ class PubServer(BaseHTTPRequestHandler): 'show announce done', 'unannounce done') + # send a newswire moderation vote from the web interface + if authorized and '/newswirevote=' in self.path and \ + self.path.startswith('/users/'): + self._newswireVote(callingDomain, self.path, + cookie, + self.server.baseDir, + self.server.httpPrefix, + self.server.domain, + self.server.domainFull, + self.server.port, + self.server.onionDomain, + self.server.i2pDomain, + GETstartTime, GETtimings, + self.server.proxyType, + self.server.debug, + self.server.newswire) + return + + # send a newswire moderation unvote from the web interface + if authorized and '/newswireunvote=' in self.path and \ + self.path.startswith('/users/'): + self._newswireUnvote(callingDomain, self.path, + cookie, + self.server.baseDir, + self.server.httpPrefix, + self.server.domain, + self.server.domainFull, + self.server.port, + self.server.onionDomain, + self.server.i2pDomain, + GETstartTime, GETtimings, + self.server.proxyType, + self.server.debug, + self.server.newswire) + return + # send a follow request approval from the web interface if authorized and '/followapprove=' in self.path and \ self.path.startswith('/users/'): @@ -9018,6 +9673,17 @@ class PubServer(BaseHTTPRequestHandler): cookie): return + # edit news post + if self._editNewsPost(callingDomain, self.path, + self.server.translate, + self.server.baseDir, + self.server.httpPrefix, + self.server.domain, + self.server.port, + self.server.domainFull, + cookie): + return + if self._showNewPost(callingDomain, self.path, self.server.mediaInstance, self.server.translate, @@ -9244,6 +9910,26 @@ class PubServer(BaseHTTPRequestHandler): 'show media 2 done', 'show blogs 2 done') + # get the news for a given person + if self.path.endswith('/tlnews') or '/tlnews?page=' in self.path: + if self._showNewsTimeline(authorized, + callingDomain, self.path, + self.server.baseDir, + self.server.httpPrefix, + self.server.domain, + self.server.domainFull, + self.server.port, + self.server.onionDomain, + self.server.i2pDomain, + GETstartTime, GETtimings, + self.server.proxyType, + cookie, self.server.debug): + return + + self._benchmarkGETtimings(GETstartTime, GETtimings, + 'show blogs 2 done', + 'show news 2 done') + # get the shared items timeline for a given person if self.path.endswith('/tlshares') or '/tlshares?page=' in self.path: if self._showSharesTimeline(authorized, @@ -10486,6 +11172,16 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline) return + if authorized and self.path.endswith('/newseditdata'): + self._newsPostEdit(callingDomain, cookie, authorized, self.path, + self.server.baseDir, self.server.httpPrefix, + self.server.domain, + self.server.domainFull, + self.server.onionDomain, + self.server.i2pDomain, self.server.debug, + self.server.defaultTimeline) + return + self._benchmarkPOSTtimings(POSTstartTime, POSTtimings, 3) # moderator action buttons @@ -11048,7 +11744,13 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None: tokensLookup[token] = nickname -def runDaemon(blogsInstance: bool, mediaInstance: bool, +def runDaemon(showPublishedDateOnly: bool, + votingTimeMins: int, + positiveVoting: bool, + newswireVotesThreshold: int, + newsInstance: bool, + blogsInstance: bool, + mediaInstance: bool, maxRecentPosts: int, enableSharedInbox: bool, registration: bool, language: str, projectVersion: str, @@ -11112,11 +11814,14 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, httpd.useBlurHash = useBlurHash httpd.mediaInstance = mediaInstance httpd.blogsInstance = blogsInstance + httpd.newsInstance = newsInstance httpd.defaultTimeline = 'inbox' if mediaInstance: httpd.defaultTimeline = 'tlmedia' if blogsInstance: httpd.defaultTimeline = 'tlblogs' + if newsInstance: + httpd.defaultTimeline = 'tlnews' # load translations dictionary httpd.translate = {} @@ -11150,6 +11855,19 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, print('ERROR: no translations loaded from ' + translationsFile) sys.exit() + # For moderated newswire feeds this is the amount of time allowed + # for voting after the post arrives + httpd.votingTimeMins = votingTimeMins + # on the newswire, whether moderators vote positively for items + # or against them (veto) + httpd.positiveVoting = positiveVoting + # number of votes needed to remove a newswire item from the news timeline + # or if positive voting is anabled to add the item to the news timeline + httpd.newswireVotesThreshold = newswireVotesThreshold + + # Show only the date at the bottom of posts, and not the time + httpd.showPublishedDateOnly = showPublishedDateOnly + if registration == 'open': httpd.registration = True else: @@ -11207,6 +11925,10 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, print('Creating shared inbox: inbox@' + domain) createSharedInbox(baseDir, 'inbox', domain, port, httpPrefix) + if not os.path.isdir(baseDir + '/accounts/news@' + domain): + print('Creating news inbox: news@' + domain) + createNewsInbox(baseDir, domain, port, httpPrefix) + if not os.path.isdir(baseDir + '/cache'): os.mkdir(baseDir + '/cache') if not os.path.isdir(baseDir + '/cache/actors'): @@ -11276,7 +11998,8 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, domainMaxPostsPerDay, accountMaxPostsPerDay, allowDeletion, debug, maxMentions, maxEmoji, httpd.translate, unitTest, - httpd.YTReplacementDomain), daemon=True) + httpd.YTReplacementDomain, + httpd.showPublishedDateOnly), daemon=True) print('Creating scheduled post thread') httpd.thrPostSchedule = \ @@ -11286,7 +12009,9 @@ def runDaemon(blogsInstance: bool, mediaInstance: bool, print('Creating newswire thread') httpd.thrNewswireDaemon = \ threadWithTrace(target=runNewswireDaemon, - args=(baseDir, httpd), daemon=True) + args=(baseDir, httpd, + httpPrefix, domain, port, + httpd.translate), daemon=True) # flags used when restarting the inbox queue httpd.restartInboxQueueInProgress = False diff --git a/epicyon-profile.css b/epicyon-profile.css index 614f114a9..d691a90c3 100644 --- a/epicyon-profile.css +++ b/epicyon-profile.css @@ -14,7 +14,8 @@ --main-header-color-roles: #282237; --main-fg-color: #dddddd; --column-left-fg-color: #dddddd; - --column-right-fg-color: #dddddd; + --column-right-fg-color: yellow; + --column-right-fg-color-voted-on: red; --main-link-color: #999; --main-link-color-hover: #bbb; --main-visited-color: #888; @@ -26,6 +27,7 @@ --font-size-button-mobile: 34px; --font-size-links: 18px; --font-size-newswire: 18px; + --font-size-newswire-mobile: 48px; --font-size: 30px; --font-size2: 24px; --font-size3: 38px; @@ -67,16 +69,21 @@ --quote-font-size: 120%; --line-spacing: 130%; --line-spacing-newswire: 100%; - --newswire-moderate-color: yellow; + --newswire-item-moderated-color: white; + --newswire-date-moderated-color: white; --column-left-width: 10vw; --column-center-width: 80vw; --column-right-width: 10vw; --column-left-header-background: #555; --column-left-header-color: #fff; --column-left-header-size: 20px; + --column-left-header-size-mobile: 50px; --column-left-icon-size: 20%; + --column-left-icon-size-mobile: 10%; + --column-left-image-width-mobile: 40vw; --column-right-icon-size: 20%; --newswire-date-color: white; + --newswire-voted-background-color: black; } @font-face { @@ -132,6 +139,11 @@ blockquote p { display: inline; } +.voteicon { + width: 1.1vw; + margin: -4px 5px; +} + .imageAnchor:focus img{ border: 2px solid var(--focus-color); } @@ -140,15 +152,6 @@ h1 { color: var(--title-color); } -h3.linksHeader { - background-color: var(--column-left-header-background); - color: var(--column-left-header-color); - font-size: var(--column-left-header-size); - text-transform: uppercase; - padding: 4px; - border: none; -} - a, u { color: var(--main-fg-color); } @@ -226,25 +229,6 @@ a:focus { width: 50%; } -.newswireItem { - font-size: var(--font-size-newswire); - color: var(--column-right-fg-color); - line-height: var(--line-spacing-newswire); -} - -.newswireItemModerate { - font-size: var(--font-size-newswire); - color: var(--newswire-moderate-color); - font-weight: bold; - line-height: var(--line-spacing-newswire); -} - -.newswireDate { - font-size: var(--font-size-newswire); - color: var(--newswire-date-color); - float: right; -} - .new-post-text { font-size: var(--font-size2); font-family: Arial, Helvetica, sans-serif; @@ -966,6 +950,53 @@ aside .toggle-inside li { font-size: var(--font-size); line-height: var(--line-spacing); } + h3.linksHeader { + background-color: var(--column-left-header-background); + color: var(--column-left-header-color); + font-size: var(--column-left-header-size); + text-transform: uppercase; + padding: 4px; + border: none; + } + .newswireItem { + font-size: var(--font-size-newswire); + color: var(--column-right-fg-color); + line-height: var(--line-spacing-newswire); + } + .newswireItemModerated { + font-size: var(--font-size-newswire); + color: var(--newswire-item-moderated-color); + line-height: var(--line-spacing-newswire); + } + .newswireDateModerated { + font-size: var(--font-size-newswire); + font-weight: bold; + color: var(--newswire-date-moderated-color); + float: right; + } + .newswireItemVotedOn a:link { + background: var(--newswire-voted-background-color); + } + .newswireItemVotedOn { + font-size: var(--font-size-newswire); + font-weight: bold; + color: var(--column-right-fg-color-voted-on); + line-height: var(--line-spacing-newswire); + } + .newswireDate { + font-size: var(--font-size-newswire); + color: var(--newswire-date-color); + float: right; + } + .newswireDateVotedOn { + font-size: var(--font-size-newswire); + font-weight: bold; + color: var(--column-right-fg-color-voted-on); + float: right; + } + .imageAnchorMobile img{ + display: none; + } .timeline-banner { background-image: linear-gradient(rgba(0, 0, 0, 0.0), rgba(0, 0, 0, 0.5)), url("banner.png"); height: 15%; @@ -1512,6 +1543,78 @@ aside .toggle-inside li { font-size: var(--font-size); line-height: var(--line-spacing); } + h3.linksHeader { + background-color: var(--column-left-header-background); + color: var(--column-left-header-color); + font-size: var(--column-left-header-size-mobile); + text-transform: uppercase; + padding: 4px; + border: none; + } + .leftColEditImage { + background: var(--main-bg-color); + width: var(--column-left-icon-size-mobile); + float: right; + margin: 20px 0px; + } + .leftColImg { + background: var(--main-bg-color); + width: var(--column-left-image-width-mobile); + float: right; + margin: 0 0; + padding: 0 0; + } + .rightColEditImage { + background: var(--main-bg-color); + width: var(--column-right-icon-size); + float: right; + margin: 20px 0px; + } + .rightColImg { + background: var(--main-bg-color); + width: 100vw; + margin: 0 0; + padding: 0 0; + } + .newswireItem { + font-size: var(--font-size-newswire-mobile); + color: var(--column-right-fg-color); + line-height: var(--line-spacing-newswire); + } + .newswireItemModerated { + font-size: var(--font-size-newswire-mobile); + color: var(--newswire-item-moderated-color); + line-height: var(--line-spacing-newswire); + } + .newswireDateModerated { + font-size: var(--font-size-newswire-mobile); + font-weight: bold; + color: var(--newswire-date-moderated-color); + float: right; + } + .newswireItemVotedOn a:link { + background: var(--newswire-voted-background-color); + } + .newswireItemVotedOn { + font-size: var(--font-size-newswire-mobile); + font-weight: bold; + color: var(--column-right-fg-color-voted-on); + line-height: var(--line-spacing-newswire); + } + .newswireDate { + font-size: var(--font-size-newswire-mobile); + color: var(--newswire-date-color); + float: right; + } + .newswireDateVotedOn { + font-size: var(--font-size-newswire-mobile); + font-weight: bold; + color: var(--column-right-fg-color-voted-on); + float: right; + } + .imageAnchorMobile img{ + display: inline; + } .timeline-banner { background-image: linear-gradient(rgba(0, 0, 0, 0.0), rgba(0, 0, 0, 0.5)), url("banner.png"); height: 6%; diff --git a/epicyon.py b/epicyon.py index 55cb19c7a..775d68682 100644 --- a/epicyon.py +++ b/epicyon.py @@ -45,10 +45,10 @@ from tests import testPostMessageBetweenServers from tests import testFollowBetweenServers from tests import testClientToServer from tests import runAllTests -from config import setConfigParam -from config import getConfigParam from auth import storeBasicCredentials from auth import createPassword +from utils import setConfigParam +from utils import getConfigParam from utils import getDomainFromActor from utils import getNicknameFromActor from utils import followPerson @@ -192,9 +192,19 @@ parser.add_argument("--noapproval", type=str2bool, nargs='?', parser.add_argument("--mediainstance", type=str2bool, nargs='?', const=True, default=False, help="Media Instance - favor media over text") +parser.add_argument("--dateonly", type=str2bool, nargs='?', + const=True, default=False, + help="Only show the date at the bottom of posts") parser.add_argument("--blogsinstance", type=str2bool, nargs='?', const=True, default=False, help="Blogs Instance - favor blogs over microblogging") +parser.add_argument("--newsinstance", type=str2bool, nargs='?', + const=True, default=False, + help="News Instance - favor news over microblogging") +parser.add_argument("--positivevoting", type=str2bool, nargs='?', + const=True, default=False, + help="On newswire, whether moderators vote " + + "positively for or veto against items") parser.add_argument("--debug", type=str2bool, nargs='?', const=True, default=False, help="Show debug messages") @@ -249,6 +259,13 @@ parser.add_argument('--archiveweeks', dest='archiveWeeks', type=str, parser.add_argument('--maxposts', dest='archiveMaxPosts', type=str, default=None, help='Maximum number of posts in in/outbox') +parser.add_argument('--minimumvotes', dest='minimumvotes', type=int, + default=1, + help='Minimum number of votes to remove or add' + + ' a newswire item') +parser.add_argument('--votingtime', dest='votingtime', type=int, + default=1440, + help='Time to vote on newswire items in minutes') parser.add_argument('--message', dest='message', type=str, default=None, help='Message content') @@ -626,6 +643,15 @@ if not args.mediainstance: args.mediainstance = mediaInstance if args.mediainstance: args.blogsinstance = False + args.newsinstance = False + +if not args.newsinstance: + newsInstance = getConfigParam(baseDir, 'newsInstance') + if newsInstance is not None: + args.newsinstance = newsInstance + if args.newsinstance: + args.blogsinstance = False + args.mediainstance = False if not args.blogsinstance: blogsInstance = getConfigParam(baseDir, 'blogsInstance') @@ -633,6 +659,7 @@ if not args.blogsinstance: args.blogsinstance = blogsInstance if args.blogsinstance: args.mediainstance = False + args.newsinstance = False # set the instance title in config.json title = getConfigParam(baseDir, 'instanceTitle') @@ -1885,6 +1912,19 @@ registration = getConfigParam(baseDir, 'registration') if not registration: registration = False +minimumvotes = getConfigParam(baseDir, 'minvotes') +if minimumvotes: + args.minimumvotes = int(minimumvotes) + +votingtime = getConfigParam(baseDir, 'votingtime') +if votingtime: + args.votingtime = votingtime + +# only show the date at the bottom of posts +dateonly = getConfigParam(baseDir, 'dateonly') +if dateonly: + args.dateonly = dateonly + YTDomain = getConfigParam(baseDir, 'youtubedomain') if YTDomain: if '://' in YTDomain: @@ -1894,11 +1934,16 @@ if YTDomain: if '.' in YTDomain: args.YTReplacementDomain = YTDomain -if setTheme(baseDir, themeName): +if setTheme(baseDir, themeName, domain): print('Theme set to ' + themeName) if __name__ == "__main__": - runDaemon(args.blogsinstance, args.mediainstance, + runDaemon(args.dateonly, + args.votingtime, + args.positivevoting, + args.minimumvotes, + args.newsinstance, + args.blogsinstance, args.mediainstance, args.maxRecentPosts, not args.nosharedinbox, registration, args.language, __version__, diff --git a/follow.py b/follow.py index dc8a7cbda..1616c0fd5 100644 --- a/follow.py +++ b/follow.py @@ -574,6 +574,10 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, print('DEBUG: follow request does not contain a ' + 'nickname for the account followed') return True + if nicknameToFollow == 'news' or nicknameToFollow == 'inbox': + if debug: + print('DEBUG: Cannot follow the news or inbox accounts') + return True handleToFollow = nicknameToFollow + '@' + domainToFollow if domainToFollow == domain: if not os.path.isdir(baseDir + '/accounts/' + handleToFollow): diff --git a/gemini/EN/install.gmi b/gemini/EN/install.gmi index 6f0b419c7..7f8f7f68b 100644 --- a/gemini/EN/install.gmi +++ b/gemini/EN/install.gmi @@ -4,7 +4,7 @@ You will need python version 3.7 or later. On a Debian based system: - sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-pyld python3-django-timezone-field python3-pyqrcode python3-png python3-bandit libimage-exiftool-perl certbot nginx + sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-django-timezone-field python3-pyqrcode python3-png python3-bandit libimage-exiftool-perl certbot nginx The following instructions install Epicyon to the /opt directory. It's not essential that it be installed there, and it could be in any other preferred directory. diff --git a/img/icons/edit_notify.png b/img/icons/edit_notify.png new file mode 100644 index 000000000..03eb4644a Binary files /dev/null and b/img/icons/edit_notify.png differ diff --git a/img/icons/hacker/edit_notify.png b/img/icons/hacker/edit_notify.png new file mode 100644 index 000000000..e0a5e991f Binary files /dev/null and b/img/icons/hacker/edit_notify.png differ diff --git a/img/icons/hacker/links.png b/img/icons/hacker/links.png new file mode 100644 index 000000000..965033d5c Binary files /dev/null and b/img/icons/hacker/links.png differ diff --git a/img/icons/hacker/logorss.png b/img/icons/hacker/logorss.png new file mode 100644 index 000000000..4d9db882c Binary files /dev/null and b/img/icons/hacker/logorss.png differ diff --git a/img/icons/hacker/newswire.png b/img/icons/hacker/newswire.png new file mode 100644 index 000000000..1ca69d635 Binary files /dev/null and b/img/icons/hacker/newswire.png differ diff --git a/img/icons/hacker/rss.png b/img/icons/hacker/rss.png deleted file mode 100644 index 48e4d74bf..000000000 Binary files a/img/icons/hacker/rss.png and /dev/null differ diff --git a/img/icons/hacker/vote.png b/img/icons/hacker/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/hacker/vote.png differ diff --git a/img/icons/henge/edit_notify.png b/img/icons/henge/edit_notify.png new file mode 100644 index 000000000..760f0afe8 Binary files /dev/null and b/img/icons/henge/edit_notify.png differ diff --git a/img/icons/henge/links.png b/img/icons/henge/links.png new file mode 100644 index 000000000..66d7f6829 Binary files /dev/null and b/img/icons/henge/links.png differ diff --git a/img/icons/henge/logorss.png b/img/icons/henge/logorss.png new file mode 100644 index 000000000..689917436 Binary files /dev/null and b/img/icons/henge/logorss.png differ diff --git a/img/icons/henge/newswire.png b/img/icons/henge/newswire.png new file mode 100644 index 000000000..f7fbc759a Binary files /dev/null and b/img/icons/henge/newswire.png differ diff --git a/img/icons/henge/rss.png b/img/icons/henge/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/henge/rss.png and /dev/null differ diff --git a/img/icons/henge/vote.png b/img/icons/henge/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/henge/vote.png differ diff --git a/img/icons/indymedia/avatar_news.png b/img/icons/indymedia/avatar_news.png new file mode 100644 index 000000000..78e8b3ed4 Binary files /dev/null and b/img/icons/indymedia/avatar_news.png differ diff --git a/img/icons/indymedia/edit_notify.png b/img/icons/indymedia/edit_notify.png new file mode 100644 index 000000000..5687aab1b Binary files /dev/null and b/img/icons/indymedia/edit_notify.png differ diff --git a/img/icons/indymedia/links.png b/img/icons/indymedia/links.png new file mode 100644 index 000000000..857c5e56d Binary files /dev/null and b/img/icons/indymedia/links.png differ diff --git a/img/icons/indymedia/logorss.png b/img/icons/indymedia/logorss.png new file mode 100644 index 000000000..38708c5d9 Binary files /dev/null and b/img/icons/indymedia/logorss.png differ diff --git a/img/icons/indymedia/newswire.png b/img/icons/indymedia/newswire.png new file mode 100644 index 000000000..190753988 Binary files /dev/null and b/img/icons/indymedia/newswire.png differ diff --git a/img/icons/indymedia/rss.png b/img/icons/indymedia/rss.png deleted file mode 100644 index e6fc9ae6d..000000000 Binary files a/img/icons/indymedia/rss.png and /dev/null differ diff --git a/img/icons/indymedia/vote.png b/img/icons/indymedia/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/indymedia/vote.png differ diff --git a/img/icons/lcd/edit_notify.png b/img/icons/lcd/edit_notify.png new file mode 100644 index 000000000..22b8d3e5c Binary files /dev/null and b/img/icons/lcd/edit_notify.png differ diff --git a/img/icons/lcd/links.png b/img/icons/lcd/links.png new file mode 100644 index 000000000..ced7f75d3 Binary files /dev/null and b/img/icons/lcd/links.png differ diff --git a/img/icons/lcd/logorss.png b/img/icons/lcd/logorss.png new file mode 100644 index 000000000..8ca232194 Binary files /dev/null and b/img/icons/lcd/logorss.png differ diff --git a/img/icons/lcd/newswire.png b/img/icons/lcd/newswire.png new file mode 100644 index 000000000..fd15cb4e9 Binary files /dev/null and b/img/icons/lcd/newswire.png differ diff --git a/img/icons/lcd/rss.png b/img/icons/lcd/rss.png deleted file mode 100644 index 1ea4c7ee9..000000000 Binary files a/img/icons/lcd/rss.png and /dev/null differ diff --git a/img/icons/lcd/vote.png b/img/icons/lcd/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/lcd/vote.png differ diff --git a/img/icons/light/edit_notify.png b/img/icons/light/edit_notify.png new file mode 100644 index 000000000..9b063fafb Binary files /dev/null and b/img/icons/light/edit_notify.png differ diff --git a/img/icons/light/links.png b/img/icons/light/links.png new file mode 100644 index 000000000..4ef5fe6a4 Binary files /dev/null and b/img/icons/light/links.png differ diff --git a/img/icons/light/logorss.png b/img/icons/light/logorss.png new file mode 100644 index 000000000..c5fad44cb Binary files /dev/null and b/img/icons/light/logorss.png differ diff --git a/img/icons/light/newswire.png b/img/icons/light/newswire.png new file mode 100644 index 000000000..f3521130d Binary files /dev/null and b/img/icons/light/newswire.png differ diff --git a/img/icons/light/rss.png b/img/icons/light/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/light/rss.png and /dev/null differ diff --git a/img/icons/light/vote.png b/img/icons/light/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/light/vote.png differ diff --git a/img/icons/links.png b/img/icons/links.png new file mode 100644 index 000000000..4ef5fe6a4 Binary files /dev/null and b/img/icons/links.png differ diff --git a/img/icons/logorss.png b/img/icons/logorss.png new file mode 100644 index 000000000..c5fad44cb Binary files /dev/null and b/img/icons/logorss.png differ diff --git a/img/icons/newswire.png b/img/icons/newswire.png new file mode 100644 index 000000000..61b0570eb Binary files /dev/null and b/img/icons/newswire.png differ diff --git a/img/icons/night/edit_notify.png b/img/icons/night/edit_notify.png new file mode 100644 index 000000000..4b7d3554a Binary files /dev/null and b/img/icons/night/edit_notify.png differ diff --git a/img/icons/night/links.png b/img/icons/night/links.png new file mode 100644 index 000000000..4ef5fe6a4 Binary files /dev/null and b/img/icons/night/links.png differ diff --git a/img/icons/night/logorss.png b/img/icons/night/logorss.png new file mode 100644 index 000000000..c5fad44cb Binary files /dev/null and b/img/icons/night/logorss.png differ diff --git a/img/icons/night/newswire.png b/img/icons/night/newswire.png new file mode 100644 index 000000000..f3521130d Binary files /dev/null and b/img/icons/night/newswire.png differ diff --git a/img/icons/night/rss.png b/img/icons/night/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/night/rss.png and /dev/null differ diff --git a/img/icons/night/vote.png b/img/icons/night/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/night/vote.png differ diff --git a/img/icons/purple/edit_notify.png b/img/icons/purple/edit_notify.png new file mode 100644 index 000000000..19b53e58a Binary files /dev/null and b/img/icons/purple/edit_notify.png differ diff --git a/img/icons/purple/links.png b/img/icons/purple/links.png new file mode 100644 index 000000000..3dfcff8e0 Binary files /dev/null and b/img/icons/purple/links.png differ diff --git a/img/icons/purple/logorss.png b/img/icons/purple/logorss.png new file mode 100644 index 000000000..3f0abfa1a Binary files /dev/null and b/img/icons/purple/logorss.png differ diff --git a/img/icons/purple/newswire.png b/img/icons/purple/newswire.png new file mode 100644 index 000000000..542a4f4c8 Binary files /dev/null and b/img/icons/purple/newswire.png differ diff --git a/img/icons/purple/rss.png b/img/icons/purple/rss.png deleted file mode 100644 index 7af046e82..000000000 Binary files a/img/icons/purple/rss.png and /dev/null differ diff --git a/img/icons/purple/vote.png b/img/icons/purple/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/purple/vote.png differ diff --git a/img/icons/rss.png b/img/icons/rss.png deleted file mode 100644 index 0b0409813..000000000 Binary files a/img/icons/rss.png and /dev/null differ diff --git a/img/icons/solidaric/edit_notify.png b/img/icons/solidaric/edit_notify.png new file mode 100644 index 000000000..f41f31d9c Binary files /dev/null and b/img/icons/solidaric/edit_notify.png differ diff --git a/img/icons/solidaric/links.png b/img/icons/solidaric/links.png new file mode 100644 index 000000000..fcfde934a Binary files /dev/null and b/img/icons/solidaric/links.png differ diff --git a/img/icons/solidaric/logorss.png b/img/icons/solidaric/logorss.png new file mode 100644 index 000000000..ac7a1c6f3 Binary files /dev/null and b/img/icons/solidaric/logorss.png differ diff --git a/img/icons/solidaric/newswire.png b/img/icons/solidaric/newswire.png new file mode 100644 index 000000000..5732a2001 Binary files /dev/null and b/img/icons/solidaric/newswire.png differ diff --git a/img/icons/solidaric/rss.png b/img/icons/solidaric/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/solidaric/rss.png and /dev/null differ diff --git a/img/icons/solidaric/vote.png b/img/icons/solidaric/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/solidaric/vote.png differ diff --git a/img/icons/starlight/edit_notify.png b/img/icons/starlight/edit_notify.png new file mode 100644 index 000000000..1faf4aed9 Binary files /dev/null and b/img/icons/starlight/edit_notify.png differ diff --git a/img/icons/starlight/links.png b/img/icons/starlight/links.png new file mode 100644 index 000000000..577d59673 Binary files /dev/null and b/img/icons/starlight/links.png differ diff --git a/img/icons/starlight/logorss.png b/img/icons/starlight/logorss.png new file mode 100644 index 000000000..90ab9a7e8 Binary files /dev/null and b/img/icons/starlight/logorss.png differ diff --git a/img/icons/starlight/newswire.png b/img/icons/starlight/newswire.png new file mode 100644 index 000000000..57b869554 Binary files /dev/null and b/img/icons/starlight/newswire.png differ diff --git a/img/icons/starlight/rss.png b/img/icons/starlight/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/starlight/rss.png and /dev/null differ diff --git a/img/icons/starlight/vote.png b/img/icons/starlight/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/starlight/vote.png differ diff --git a/img/icons/vote.png b/img/icons/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/vote.png differ diff --git a/img/icons/zen/edit_notify.png b/img/icons/zen/edit_notify.png new file mode 100644 index 000000000..627daa8b8 Binary files /dev/null and b/img/icons/zen/edit_notify.png differ diff --git a/img/icons/zen/links.png b/img/icons/zen/links.png new file mode 100644 index 000000000..fcfde934a Binary files /dev/null and b/img/icons/zen/links.png differ diff --git a/img/icons/zen/logorss.png b/img/icons/zen/logorss.png new file mode 100644 index 000000000..64257681b Binary files /dev/null and b/img/icons/zen/logorss.png differ diff --git a/img/icons/zen/rss.png b/img/icons/zen/rss.png deleted file mode 100644 index 533453fc7..000000000 Binary files a/img/icons/zen/rss.png and /dev/null differ diff --git a/img/icons/zen/vote.png b/img/icons/zen/vote.png new file mode 100644 index 000000000..c810092b0 Binary files /dev/null and b/img/icons/zen/vote.png differ diff --git a/img/options_background_indymedia.jpg b/img/options_background_indymedia.jpg index 1e2ce54df..e0038e4b9 100644 Binary files a/img/options_background_indymedia.jpg and b/img/options_background_indymedia.jpg differ diff --git a/img/right_col_image_purple.png b/img/right_col_image_purple.png new file mode 100644 index 000000000..2797c7db3 Binary files /dev/null and b/img/right_col_image_purple.png differ diff --git a/inbox.py b/inbox.py index c61b9320d..0fb51d0a6 100644 --- a/inbox.py +++ b/inbox.py @@ -129,20 +129,22 @@ def inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, session, cachedWebfingers: {}, personCache: {}, nickname: str, domain: str, port: int, postJsonObject: {}, - allowDeletion: bool, boxname: str) -> None: + allowDeletion: bool, boxname: str, + showPublishedDateOnly: bool) -> None: """Converts the json post into html and stores it in a cache This enables the post to be quickly displayed later """ pageNumber = -999 avatarUrl = None if boxname != 'tlevents' and boxname != 'outbox': - boxName = 'inbox' + boxname = 'inbox' individualPostAsHtml(True, recentPostsCache, maxRecentPosts, getIconsDir(baseDir), translate, pageNumber, baseDir, session, cachedWebfingers, personCache, nickname, domain, port, postJsonObject, avatarUrl, True, allowDeletion, - httpPrefix, __version__, boxName, + httpPrefix, __version__, boxname, None, + showPublishedDateOnly, not isDM(postJsonObject), True, True, False, True) @@ -2027,7 +2029,8 @@ def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, queueFilename: str, destinationFilename: str, maxReplies: int, allowDeletion: bool, maxMentions: int, maxEmoji: int, translate: {}, - unitTest: bool, YTReplacementDomain: str) -> bool: + unitTest: bool, YTReplacementDomain: str, + showPublishedDateOnly: bool) -> bool: """ Anything which needs to be done after initial checks have passed """ actor = keyId @@ -2235,13 +2238,23 @@ def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, sendingActorDomain, sendingActorPort = \ getDomainFromActor(sendingActor) if sendingActorNickname and sendingActorDomain: + if not os.path.isfile(followingFilename): + print('No following.txt file exists for ' + + nickname + '@' + domain + + ' so not accepting DM from ' + + sendingActorNickname + '@' + + sendingActorDomain) + return False sendH = \ sendingActorNickname + '@' + sendingActorDomain if sendH != nickname + '@' + domain: - if sendH not in open(followingFilename).read(): + if sendH not in \ + open(followingFilename).read(): print(nickname + '@' + domain + - ' cannot receive DM from ' + sendH + - ' because they do not follow them') + ' cannot receive DM from ' + + sendH + + ' because they do not ' + + 'follow them') return False else: return False @@ -2327,7 +2340,8 @@ def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, domain, port, postJsonObject, allowDeletion, - boxname) + boxname, + showPublishedDateOnly) if debug: timeDiff = \ str(int((time.time() - htmlCacheStartTime) * @@ -2421,7 +2435,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, domainMaxPostsPerDay: int, accountMaxPostsPerDay: int, allowDeletion: bool, debug: bool, maxMentions: int, maxEmoji: int, translate: {}, unitTest: bool, - YTReplacementDomain: str) -> None: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> None: """Processes received items and moves them to the appropriate directories """ @@ -2833,7 +2848,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, maxReplies, allowDeletion, maxMentions, maxEmoji, translate, unitTest, - YTReplacementDomain) + YTReplacementDomain, + showPublishedDateOnly) if debug: pprint(queueJson['post']) diff --git a/jsonldsig.py b/jsonldsig.py index 3e3761969..07f6c126d 100644 --- a/jsonldsig.py +++ b/jsonldsig.py @@ -22,7 +22,7 @@ except ImportError: from Crypto.Hash import SHA256 from Crypto.Signature import PKCS1_v1_5 -from pyld import jsonld +from pyjsonld import normalize import base64 import json @@ -107,7 +107,7 @@ def jsonldNormalize(jldDocument: str): 'algorithm': 'URDNA2015', 'format': 'application/nquads' } - normalized = jsonld.normalize(jldDocument, options=options) + normalized = normalize(jldDocument, options=options) normalizedHash = SHA256.new(data=normalized.encode('utf-8')).digest() return normalizedHash diff --git a/newsdaemon.py b/newsdaemon.py new file mode 100644 index 000000000..d4d63ee40 --- /dev/null +++ b/newsdaemon.py @@ -0,0 +1,273 @@ +__filename__ = "newsdaemon.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "1.1.0" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import os +import time +import datetime +from collections import OrderedDict +from newswire import getDictFromNewswire +from posts import createNewsPost +from content import removeHtmlTag +from content import dangerousMarkup +from utils import loadJson +from utils import saveJson +from utils import getStatusNumber + + +def updateFeedsOutboxIndex(baseDir: str, domain: str, postId: str) -> None: + """Updates the index used for imported RSS feeds + """ + basePath = baseDir + '/accounts/news@' + domain + indexFilename = basePath + '/outbox.index' + + if os.path.isfile(indexFilename): + if postId not in open(indexFilename).read(): + try: + with open(indexFilename, 'r+') as feedsFile: + content = feedsFile.read() + feedsFile.seek(0, 0) + feedsFile.write(postId + '\n' + content) + print('DEBUG: feeds post added to index') + except Exception as e: + print('WARN: Failed to write entry to feeds posts index ' + + indexFilename + ' ' + str(e)) + else: + feedsFile = open(indexFilename, 'w+') + if feedsFile: + feedsFile.write(postId + '\n') + feedsFile.close() + + +def saveArrivedTime(baseDir: str, postFilename: str, arrived: str) -> None: + """Saves the time when an rss post arrived to a file + """ + arrivedFile = open(postFilename + '.arrived', 'w+') + if arrivedFile: + arrivedFile.write(arrived) + arrivedFile.close() + + +def removeControlCharacters(content: str) -> str: + """TODO this is hacky and a better solution is needed + the unicode is messing up somehow + """ + lookups = { + "8211": "-", + "8230": "...", + "8216": "'", + "8217": "'", + "8220": '"', + "8221": '"' + } + for code, ch in lookups.items(): + content = content.replace('&' + code + ';', ch) + content = content.replace('&#' + code + ';', ch) + return content + + +def convertRSStoActivityPub(baseDir: str, httpPrefix: str, + domain: str, port: int, + newswire: {}, + translate: {}, + recentPostsCache: {}, maxRecentPosts: int, + session, cachedWebfingers: {}, + personCache: {}) -> None: + """Converts rss items in a newswire into posts + """ + basePath = baseDir + '/accounts/news@' + domain + '/outbox' + if not os.path.isdir(basePath): + os.mkdir(basePath) + + # oldest items first + newswireReverse = \ + OrderedDict(sorted(newswire.items(), reverse=False)) + + for dateStr, item in newswireReverse.items(): + originalDateStr = dateStr + # convert the date to the format used by ActivityPub + dateStr = dateStr.replace(' ', 'T') + dateStr = dateStr.replace('+00:00', 'Z') + + statusNumber, published = getStatusNumber(dateStr) + newPostId = \ + httpPrefix + '://' + domain + \ + '/users/news/statuses/' + statusNumber + + # file where the post is stored + filename = basePath + '/' + newPostId.replace('/', '#') + '.json' + if os.path.isfile(filename): + # don't create the post if it already exists + # set the url + newswire[originalDateStr][1] = \ + '/users/news/statuses/' + statusNumber + # set the filename + newswire[originalDateStr][3] = filename + continue + + rssTitle = removeControlCharacters(item[0]) + url = item[1] + if dangerousMarkup(url) or dangerousMarkup(rssTitle): + continue + rssDescription = '' + + # get the rss description if it exists + rssDescription = removeControlCharacters(item[4]) + if rssDescription.startswith('', '') + rssDescription = '

' + rssDescription + '

' + + # add the off-site link to the description + if rssDescription and not dangerousMarkup(rssDescription): + rssDescription += \ + '
' + \ + translate['Read more...'] + '' + else: + rssDescription = \ + '' + \ + translate['Read more...'] + '' + + # remove image dimensions + if ' None: + """Preserve any votes or generated activitypub post filename + as rss feeds are updated + """ + for published, fields in oldNewswire.items(): + if not newNewswire.get(published): + continue + newNewswire[published][1] = fields[1] + newNewswire[published][2] = fields[2] + newNewswire[published][3] = fields[3] + + +def runNewswireDaemon(baseDir: str, httpd, + httpPrefix: str, domain: str, port: int, + translate: {}) -> None: + """Periodically updates RSS feeds + """ + newswireStateFilename = baseDir + '/accounts/.newswirestate.json' + + # initial sleep to allow the system to start up + time.sleep(50) + while True: + # has the session been created yet? + if not httpd.session: + print('Newswire daemon waiting for session') + time.sleep(60) + continue + + # try to update the feeds + newNewswire = None + try: + newNewswire = getDictFromNewswire(httpd.session, baseDir) + except Exception as e: + print('WARN: unable to update newswire ' + str(e)) + time.sleep(120) + continue + + if not httpd.newswire: + if os.path.isfile(newswireStateFilename): + httpd.newswire = loadJson(newswireStateFilename) + + mergeWithPreviousNewswire(httpd.newswire, newNewswire) + + httpd.newswire = newNewswire + saveJson(httpd.newswire, newswireStateFilename) + print('Newswire updated') + + convertRSStoActivityPub(baseDir, + httpPrefix, domain, port, + newNewswire, translate, + httpd.recentPostsCache, + httpd.maxRecentPosts, + httpd.session, + httpd.cachedWebfingers, + httpd.personCache) + print('Newswire feed converted to ActivityPub') + + # wait a while before the next feeds update + time.sleep(1200) + + +def runNewswireWatchdog(projectVersion: str, httpd) -> None: + """This tries to keep the newswire update thread running even if it dies + """ + print('Starting newswire watchdog') + newswireOriginal = \ + httpd.thrPostSchedule.clone(runNewswireDaemon) + httpd.thrNewswireDaemon.start() + while True: + time.sleep(50) + if not httpd.thrNewswireDaemon.isAlive(): + httpd.thrNewswireDaemon.kill() + httpd.thrNewswireDaemon = \ + newswireOriginal.clone(runNewswireDaemon) + httpd.thrNewswireDaemon.start() + print('Restarting newswire daemon...') diff --git a/newswire.py b/newswire.py index bda5e8b06..2d3045b20 100644 --- a/newswire.py +++ b/newswire.py @@ -7,7 +7,6 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os -import time import requests from socket import error as SocketError import errno @@ -15,11 +14,15 @@ from datetime import datetime from collections import OrderedDict from utils import locatePost from utils import loadJson +from utils import saveJson +from utils import isSuspended def rss2Header(httpPrefix: str, nickname: str, domainFull: str, title: str, translate: {}) -> str: + """Header for an RSS 2.0 feed + """ rssStr = "" rssStr += "" rssStr += '' @@ -37,12 +40,14 @@ def rss2Header(httpPrefix: str, def rss2Footer() -> str: + """Footer for an RSS 2.0 feed + """ rssStr = '' rssStr += '' return rssStr -def xml2StrToDict(xmlStr: str) -> {}: +def xml2StrToDict(xmlStr: str, moderated: bool) -> {}: """Converts an xml 2.0 string to a dictionary """ if '' not in xmlStr: @@ -64,6 +69,10 @@ def xml2StrToDict(xmlStr: str) -> {}: continue title = rssItem.split('')[1] title = title.split('')[0] + description = '' + if '' in rssItem and '' in rssItem: + description = rssItem.split('')[1] + description = description.split('')[0] link = rssItem.split('')[1] link = link.split('')[0] pubDate = rssItem.split('')[1] @@ -72,7 +81,11 @@ def xml2StrToDict(xmlStr: str) -> {}: try: publishedDate = \ datetime.strptime(pubDate, "%a, %d %b %Y %H:%M:%S %z") - result[str(publishedDate)] = [title, link] + postFilename = '' + votesStatus = [] + result[str(publishedDate)] = [title, link, + votesStatus, postFilename, + description, moderated] parsed = True except BaseException: pass @@ -88,15 +101,71 @@ def xml2StrToDict(xmlStr: str) -> {}: return result -def xmlStrToDict(xmlStr: str) -> {}: +def atomFeedToDict(xmlStr: str, moderated: bool) -> {}: + """Converts an atom feed string to a dictionary + """ + if '' not in xmlStr: + return {} + result = {} + rssItems = xmlStr.split('') + for rssItem in rssItems: + if '' not in rssItem: + continue + if '' not in rssItem: + continue + if '' not in rssItem: + continue + if '' not in rssItem: + continue + if '' not in rssItem: + continue + if '' not in rssItem: + continue + title = rssItem.split('')[1] + title = title.split('')[0] + description = '' + if '

' in rssItem and '' in rssItem: + description = rssItem.split('')[1] + description = description.split('')[0] + link = rssItem.split('')[1] + link = link.split('')[0] + pubDate = rssItem.split('')[1] + pubDate = pubDate.split('')[0] + parsed = False + try: + publishedDate = \ + datetime.strptime(pubDate, "%Y-%m-%dT%H:%M:%SZ") + postFilename = '' + votesStatus = [] + result[str(publishedDate)] = [title, link, + votesStatus, postFilename, + description, moderated] + parsed = True + except BaseException: + pass + if not parsed: + try: + publishedDate = \ + datetime.strptime(pubDate, "%a, %d %b %Y %H:%M:%S UT") + result[str(publishedDate) + '+00:00'] = [title, link] + parsed = True + except BaseException: + print('WARN: unrecognized atom feed date format: ' + pubDate) + pass + return result + + +def xmlStrToDict(xmlStr: str, moderated: bool) -> {}: """Converts an xml string to a dictionary """ if 'rss version="2.0"' in xmlStr: - return xml2StrToDict(xmlStr) + return xml2StrToDict(xmlStr, moderated) + elif 'xmlns="http://www.w3.org/2005/Atom"' in xmlStr: + return atomFeedToDict(xmlStr, moderated) return {} -def getRSS(session, url: str) -> {}: +def getRSS(session, url: str, moderated: bool) -> {}: """Returns an RSS url as a dict """ if not isinstance(url, str): @@ -119,7 +188,7 @@ def getRSS(session, url: str) -> {}: print('WARN: no session specified for getRSS') try: result = session.get(url, headers=sessionHeaders, params=sessionParams) - return xmlStrToDict(result.text) + return xmlStrToDict(result.text, moderated) except requests.exceptions.RequestException as e: print('ERROR: getRSS failed\nurl: ' + str(url) + '\n' + 'headers: ' + str(sessionHeaders) + '\n' + @@ -155,7 +224,10 @@ def getRSSfromDict(baseDir: str, newswire: {}, continue rssStr += '\n' rssStr += ' ' + fields[0] + '\n' - rssStr += ' ' + fields[1] + '\n' + url = fields[1] + if domainFull not in url: + url = httpPrefix + '://' + domainFull + url + rssStr += ' ' + url + '\n' rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT") rssStr += ' ' + rssDateStr + '\n' @@ -164,6 +236,22 @@ def getRSSfromDict(baseDir: str, newswire: {}, return rssStr +def isaBlogPost(postJsonObject: {}) -> bool: + """Is the given object a blog post? + """ + if not postJsonObject: + return False + if not postJsonObject.get('object'): + return False + if not isinstance(postJsonObject['object'], dict): + return False + if postJsonObject['object'].get('summary') and \ + postJsonObject['object'].get('url') and \ + postJsonObject['object'].get('published'): + return True + return False + + def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, newswire: {}, maxBlogsPerAccount: int, @@ -172,6 +260,16 @@ def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, """ if not os.path.isfile(indexFilename): return + # local blog entries are unmoderated by default + moderated = False + + # local blogs can potentially be moderated + moderatedFilename = \ + baseDir + '/accounts/' + nickname + '@' + domain + \ + '/.newswiremoderated' + if os.path.isfile(moderatedFilename): + moderated = True + with open(indexFilename, 'r') as indexFile: postFilename = 'start' ctr = 0 @@ -193,43 +291,39 @@ def addAccountBlogsToNewswire(baseDir: str, nickname: str, domain: str, fullPostFilename = \ locatePost(baseDir, nickname, domain, postUrl, False) - isAPost = False + if not fullPostFilename: + print('Unable to locate post ' + postUrl) + ctr += 1 + if ctr >= maxBlogsPerAccount: + break + continue + postJsonObject = None if fullPostFilename: postJsonObject = loadJson(fullPostFilename) - if postJsonObject: - if postJsonObject.get('object'): - if isinstance(postJsonObject['object'], dict): - isAPost = True - if isAPost: - if postJsonObject['object'].get('summary') and \ - postJsonObject['object'].get('url') and \ - postJsonObject['object'].get('published'): - published = postJsonObject['object']['published'] - published = published.replace('T', ' ') - published = published.replace('Z', '+00:00') - newswire[published] = \ - [postJsonObject['object']['summary'], - postJsonObject['object']['url']] + if isaBlogPost(postJsonObject): + published = postJsonObject['object']['published'] + published = published.replace('T', ' ') + published = published.replace('Z', '+00:00') + votes = [] + if os.path.isfile(fullPostFilename + '.votes'): + votes = loadJson(fullPostFilename + '.votes') + description = '' + newswire[published] = \ + [postJsonObject['object']['summary'], + postJsonObject['object']['url'], votes, + fullPostFilename, description, moderated] ctr += 1 if ctr >= maxBlogsPerAccount: break -def addLocalBlogsToNewswire(baseDir: str, newswire: {}, - maxBlogsPerAccount: int) -> None: - """Adds blogs from this instance into the newswire +def addBlogsToNewswire(baseDir: str, newswire: {}, + maxBlogsPerAccount: int) -> None: + """Adds blogs from each user account into the newswire """ - # get the list of handles who are trusted to post to the newswire - newswireTrusted = '' - newswireTrustedFilename = baseDir + '/accounts/newswiretrusted.txt' - if os.path.isfile(newswireTrustedFilename): - with open(newswireTrustedFilename, "r") as trustFile: - newswireTrusted = trustFile.read() - - # file containing suspended account nicknames - suspendedFilename = baseDir + '/accounts/suspended.txt' + moderationDict = {} # go through each account for subdir, dirs, files in os.walk(baseDir + '/accounts'): @@ -238,25 +332,19 @@ def addLocalBlogsToNewswire(baseDir: str, newswire: {}, continue if 'inbox@' in handle: continue - if handle not in newswireTrusted: - if handle.split('@')[0] + '\n' not in newswireTrusted: - continue - accountDir = os.path.join(baseDir + '/accounts', handle) + + nickname = handle.split('@')[0] # has this account been suspended? - nickname = handle.split('@')[0] - if os.path.isfile(suspendedFilename): - with open(suspendedFilename, "r") as f: - lines = f.readlines() - foundSuspended = False - for nick in lines: - if nick == nickname + '\n': - foundSuspended = True - break - if foundSuspended: - continue + if isSuspended(baseDir, nickname): + continue + + if os.path.isfile(baseDir + '/accounts/' + handle + + '/.nonewswire'): + continue # is there a blogs timeline for this account? + accountDir = os.path.join(baseDir + '/accounts', handle) blogsIndex = accountDir + '/tlblogs.index' if os.path.isfile(blogsIndex): domain = handle.split('@')[1] @@ -264,6 +352,18 @@ def addLocalBlogsToNewswire(baseDir: str, newswire: {}, newswire, maxBlogsPerAccount, blogsIndex) + # sort the moderation dict into chronological order, latest first + sortedModerationDict = \ + OrderedDict(sorted(moderationDict.items(), reverse=True)) + # save the moderation queue details for later display + newswireModerationFilename = baseDir + '/accounts/newswiremoderation.txt' + if sortedModerationDict: + saveJson(sortedModerationDict, newswireModerationFilename) + else: + # remove the file if there is nothing to moderate + if os.path.isfile(newswireModerationFilename): + os.remove(newswireModerationFilename) + def getDictFromNewswire(session, baseDir: str) -> {}: """Gets rss feeds as a dictionary from newswire file @@ -279,61 +379,28 @@ def getDictFromNewswire(session, baseDir: str) -> {}: result = {} for url in rssFeed: url = url.strip() + + # Does this contain a url? if '://' not in url: continue + + # is this a comment? if url.startswith('#'): continue - itemsList = getRSS(session, url) + + # should this feed be moderated? + moderated = False + if '*' in url: + moderated = True + url = url.replace('*', '').strip() + + itemsList = getRSS(session, url, moderated) for dateStr, item in itemsList.items(): result[dateStr] = item - # add local content - addLocalBlogsToNewswire(baseDir, result, 5) + # add blogs from each user account + addBlogsToNewswire(baseDir, result, 5) # sort into chronological order, latest first sortedResult = OrderedDict(sorted(result.items(), reverse=True)) return sortedResult - - -def runNewswireDaemon(baseDir: str, httpd): - """Periodically updates RSS feeds - """ - # initial sleep to allow the system to start up - time.sleep(70) - while True: - # has the session been created yet? - if not httpd.session: - print('Newswire daemon waiting for session') - time.sleep(60) - continue - - # try to update the feeds - newNewswire = None - try: - newNewswire = getDictFromNewswire(httpd.session, baseDir) - except BaseException: - print('WARN: unable to update newswire') - time.sleep(120) - continue - - httpd.newswire = newNewswire - print('Newswire updated') - # wait a while before the next feeds update - time.sleep(1200) - - -def runNewswireWatchdog(projectVersion: str, httpd) -> None: - """This tries to keep the newswire update thread running even if it dies - """ - print('Starting newswire watchdog') - newswireOriginal = \ - httpd.thrPostSchedule.clone(runNewswireDaemon) - httpd.thrNewswireDaemon.start() - while True: - time.sleep(50) - if not httpd.thrNewswireDaemon.isAlive(): - httpd.thrNewswireDaemon.kill() - httpd.thrNewswireDaemon = \ - newswireOriginal.clone(runNewswireDaemon) - httpd.thrNewswireDaemon.start() - print('Restarting newswire daemon...') diff --git a/outbox.py b/outbox.py index 56016b683..1329855a6 100644 --- a/outbox.py +++ b/outbox.py @@ -45,7 +45,8 @@ def postMessageToOutbox(messageJson: {}, postToNickname: str, postLog: [], cachedWebfingers: {}, personCache: {}, allowDeletion: bool, proxyType: str, version: str, debug: bool, - YTReplacementDomain: str) -> bool: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> bool: """post is received by the outbox Client to server message post https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery diff --git a/person.py b/person.py index d00724294..f4539e180 100644 --- a/person.py +++ b/person.py @@ -23,6 +23,7 @@ from webfinger import storeWebfingerEndpoint from posts import createDMTimeline from posts import createRepliesTimeline from posts import createMediaTimeline +from posts import createNewsTimeline from posts import createBlogsTimeline from posts import createBookmarksTimeline from posts import createEventsTimeline @@ -34,11 +35,10 @@ from auth import removePassword from roles import setRole from media import removeMetaData from utils import validNickname -from utils import noOfAccounts from utils import loadJson from utils import saveJson -from config import setConfigParam -from config import getConfigParam +from utils import setConfigParam +from utils import getConfigParam def generateRSAKey() -> (str, str): @@ -444,12 +444,13 @@ def createPerson(baseDir: str, nickname: str, domain: str, port: int, saveToFile, manualFollowerApproval, password) - if noOfAccounts(baseDir) == 1: + if not getConfigParam(baseDir, 'admin'): # print(nickname+' becomes the instance admin and a moderator') + setConfigParam(baseDir, 'admin', nickname) setRole(baseDir, nickname, domain, 'instance', 'admin') setRole(baseDir, nickname, domain, 'instance', 'moderator') + setRole(baseDir, nickname, domain, 'instance', 'editor') setRole(baseDir, nickname, domain, 'instance', 'delegator') - setConfigParam(baseDir, 'admin', nickname) if not os.path.isdir(baseDir + '/accounts'): os.mkdir(baseDir + '/accounts') @@ -503,6 +504,14 @@ def createSharedInbox(baseDir: str, nickname: str, domain: str, port: int, True, True, None) +def createNewsInbox(baseDir: str, domain: str, port: int, + httpPrefix: str) -> (str, str, {}, {}): + """Generates the news inbox + """ + return createPersonBase(baseDir, 'news', domain, port, httpPrefix, + True, True, None) + + def personUpgradeActor(baseDir: str, personJson: {}, handle: str, filename: str) -> None: """Alter the actor to add any new properties @@ -586,12 +595,14 @@ def personLookup(domain: str, path: str, baseDir: str) -> {}: def personBoxJson(recentPostsCache: {}, session, baseDir: str, domain: str, port: int, path: str, httpPrefix: str, noOfItems: int, boxname: str, - authorized: bool) -> {}: + authorized: bool, + newswireVotesThreshold: int, positiveVoting: bool, + votingTimeMins: int) -> {}: """Obtain the inbox/outbox/moderation feed for the given person """ if boxname != 'inbox' and boxname != 'dm' and \ boxname != 'tlreplies' and boxname != 'tlmedia' and \ - boxname != 'tlblogs' and \ + boxname != 'tlblogs' and boxname != 'tlnews' and \ boxname != 'outbox' and boxname != 'moderation' and \ boxname != 'tlbookmarks' and boxname != 'bookmarks' and \ boxname != 'tlevents': @@ -659,6 +670,11 @@ def personBoxJson(recentPostsCache: {}, return createMediaTimeline(session, baseDir, nickname, domain, port, httpPrefix, noOfItems, headerOnly, pageNumber) + elif boxname == 'tlnews': + return createNewsTimeline(session, baseDir, nickname, domain, port, + httpPrefix, noOfItems, headerOnly, + newswireVotesThreshold, positiveVoting, + votingTimeMins, pageNumber) elif boxname == 'tlblogs': return createBlogsTimeline(session, baseDir, nickname, domain, port, httpPrefix, noOfItems, headerOnly, @@ -754,23 +770,6 @@ def setBio(baseDir: str, nickname: str, domain: str, bio: str) -> bool: return True -def isSuspended(baseDir: str, nickname: str) -> bool: - """Returns true if the given nickname is suspended - """ - adminNickname = getConfigParam(baseDir, 'admin') - if nickname == adminNickname: - return False - - suspendedFilename = baseDir + '/accounts/suspended.txt' - if os.path.isfile(suspendedFilename): - with open(suspendedFilename, "r") as f: - lines = f.readlines() - for suspended in lines: - if suspended.strip('\n').strip('\r') == nickname: - return True - return False - - def unsuspendAccount(baseDir: str, nickname: str) -> None: """Removes an account suspention """ @@ -790,6 +789,8 @@ def suspendAccount(baseDir: str, nickname: str, domain: str) -> None: """ # Don't suspend the admin adminNickname = getConfigParam(baseDir, 'admin') + if not adminNickname: + return if nickname == adminNickname: return @@ -844,6 +845,8 @@ def canRemovePost(baseDir: str, nickname: str, # is the post by the admin? adminNickname = getConfigParam(baseDir, 'admin') + if not adminNickname: + return False if domainFull + '/users/' + adminNickname + '/' in postId: return False @@ -900,6 +903,8 @@ def removeAccount(baseDir: str, nickname: str, """ # Don't remove the admin adminNickname = getConfigParam(baseDir, 'admin') + if not adminNickname: + return False if nickname == adminNickname: return False diff --git a/posts.py b/posts.py index 53c5d3bae..8ef55044b 100644 --- a/posts.py +++ b/posts.py @@ -45,6 +45,10 @@ from utils import validNickname from utils import locatePost from utils import loadJson from utils import saveJson +from utils import getConfigParam +from utils import locateNewsVotes +from utils import locateNewsArrival +from utils import votesOnNewswireItem from media import attachMedia from media import replaceYouTube from content import removeHtml @@ -53,16 +57,11 @@ from content import addHtmlTags from content import replaceEmojiFromTags from content import removeTextFormatting from auth import createBasicAuthHeader -from config import getConfigParam from blocking import isBlocked from filters import isFiltered from git import convertPostToPatch from jsonldsig import jsonldSign from petnames import resolvePetnames -# try: -# from BeautifulSoup import BeautifulSoup -# except ImportError: -# from bs4 import BeautifulSoup def isModerator(baseDir: str, nickname: str) -> bool: @@ -71,14 +70,20 @@ def isModerator(baseDir: str, nickname: str) -> bool: moderatorsFile = baseDir + '/accounts/moderators.txt' if not os.path.isfile(moderatorsFile): - if getConfigParam(baseDir, 'admin') == nickname: + adminName = getConfigParam(baseDir, 'admin') + if not adminName: + return False + if adminName == nickname: return True return False with open(moderatorsFile, "r") as f: lines = f.readlines() if len(lines) == 0: - if getConfigParam(baseDir, 'admin') == nickname: + adminName = getConfigParam(baseDir, 'admin') + if not adminName: + return False + if adminName == nickname: return True for moderator in lines: moderator = moderator.strip('\n').strip('\r') @@ -87,6 +92,34 @@ def isModerator(baseDir: str, nickname: str) -> bool: return False +def isEditor(baseDir: str, nickname: str) -> bool: + """Returns true if the given nickname is an editor + """ + editorsFile = baseDir + '/accounts/editors.txt' + + if not os.path.isfile(editorsFile): + adminName = getConfigParam(baseDir, 'admin') + if not adminName: + return False + if adminName == nickname: + return True + return False + + with open(editorsFile, "r") as f: + lines = f.readlines() + if len(lines) == 0: + adminName = getConfigParam(baseDir, 'admin') + if not adminName: + return False + if adminName == nickname: + return True + for editor in lines: + editor = editor.strip('\n').strip('\r') + if editor == nickname: + return True + return False + + def noOfFollowersOnDomain(baseDir: str, handle: str, domain: str, followFile='followers.txt') -> int: """Returns the number of followers of the given handle from the given domain @@ -505,7 +538,8 @@ def deleteAllPosts(baseDir: str, """Deletes all posts for a person from inbox or outbox """ if boxname != 'inbox' and boxname != 'outbox' and \ - boxname != 'tlblogs' and boxname != 'tlevents': + boxname != 'tlblogs' and boxname != 'tlnews' and \ + boxname != 'tlevents': return boxDir = createPersonDir(nickname, domain, baseDir, boxname) for deleteFilename in os.scandir(boxDir): @@ -527,7 +561,8 @@ def savePostToBox(baseDir: str, httpPrefix: str, postId: str, Returns the filename """ if boxname != 'inbox' and boxname != 'outbox' and \ - boxname != 'tlblogs' and boxname != 'tlevents' and \ + boxname != 'tlblogs' and boxname != 'tlnews' and \ + boxname != 'tlevents' and \ boxname != 'scheduled': return None originalDomain = domain @@ -722,8 +757,11 @@ def createPostBase(baseDir: str, nickname: str, domain: str, port: int, """ subject = addAutoCW(baseDir, nickname, domain, subject, content) - mentionedRecipients = \ - getMentionedPeople(baseDir, httpPrefix, content, domain, False) + if nickname != 'news': + mentionedRecipients = \ + getMentionedPeople(baseDir, httpPrefix, content, domain, False) + else: + mentionedRecipients = '' tags = [] hashtagsDict = {} @@ -734,18 +772,20 @@ def createPostBase(baseDir: str, nickname: str, domain: str, port: int, domain = domain + ':' + str(port) # add tags - content = \ - addHtmlTags(baseDir, httpPrefix, - nickname, domain, content, - mentionedRecipients, - hashtagsDict, True) + if nickname != 'news': + content = \ + addHtmlTags(baseDir, httpPrefix, + nickname, domain, content, + mentionedRecipients, + hashtagsDict, True) # replace emoji with unicode tags = [] for tagName, tag in hashtagsDict.items(): tags.append(tag) # get list of tags - content = replaceEmojiFromTags(content, tags, 'content') + if nickname != 'news': + content = replaceEmojiFromTags(content, tags, 'content') # remove replaced emoji hashtagsDictCopy = hashtagsDict.copy() for tagName, tag in hashtagsDictCopy.items(): @@ -1177,11 +1217,39 @@ def createBlogPost(baseDir: str, inReplyTo=None, inReplyToAtomUri=None, subject=None, schedulePost=False, eventDate=None, eventTime=None, location=None) -> {}: + commentsEnabled = True blog = \ createPublicPost(baseDir, nickname, domain, port, httpPrefix, content, followersOnly, saveToFile, - clientToServer, + clientToServer, commentsEnabled, + attachImageFilename, mediaType, + imageDescription, useBlurhash, + inReplyTo, inReplyToAtomUri, subject, + schedulePost, + eventDate, eventTime, location) + blog['object']['type'] = 'Article' + return blog + + +def createNewsPost(baseDir: str, + domain: str, port: int, httpPrefix: str, + content: str, followersOnly: bool, saveToFile: bool, + attachImageFilename: str, mediaType: str, + imageDescription: str, useBlurhash: bool, + subject: str) -> {}: + clientToServer = False + inReplyTo = None + inReplyToAtomUri = None + schedulePost = False + eventDate = None + eventTime = None + location = None + blog = \ + createPublicPost(baseDir, + 'news', domain, port, httpPrefix, + content, followersOnly, saveToFile, + clientToServer, False, attachImageFilename, mediaType, imageDescription, useBlurhash, inReplyTo, inReplyToAtomUri, subject, @@ -1688,8 +1756,8 @@ def sendPost(projectVersion: str, try: signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem) postJsonObject = signedPostJsonObject - except BaseException: - print('WARN: failed to JSON-LD sign post') + except Exception as e: + print('WARN: failed to JSON-LD sign post, ' + str(e)) pass # convert json to string so that there are no @@ -2027,8 +2095,8 @@ def sendSignedJson(postJsonObject: {}, session, baseDir: str, try: signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem) postJsonObject = signedPostJsonObject - except BaseException: - print('WARN: failed to JSON-LD sign post') + except Exception as e: + print('WARN: failed to JSON-LD sign post, ' + str(e)) pass # convert json to string so that there are no @@ -2444,7 +2512,7 @@ def createInbox(recentPostsCache: {}, session, baseDir, 'inbox', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - pageNumber) + 0, False, 0, pageNumber) def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2453,7 +2521,7 @@ def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlbookmarks', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, - True, pageNumber) + True, 0, False, 0, pageNumber) def createEventsTimeline(recentPostsCache: {}, @@ -2463,7 +2531,7 @@ def createEventsTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'tlevents', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, - True, pageNumber) + True, 0, False, 0, pageNumber) def createDMTimeline(recentPostsCache: {}, @@ -2473,7 +2541,7 @@ def createDMTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'dm', nickname, domain, port, httpPrefix, itemsPerPage, - headerOnly, True, pageNumber) + headerOnly, True, 0, False, 0, pageNumber) def createRepliesTimeline(recentPostsCache: {}, @@ -2483,7 +2551,7 @@ def createRepliesTimeline(recentPostsCache: {}, return createBoxIndexed(recentPostsCache, session, baseDir, 'tlreplies', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - pageNumber) + 0, False, 0, pageNumber) def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2492,7 +2560,7 @@ def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlblogs', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - pageNumber) + 0, False, 0, pageNumber) def createMediaTimeline(session, baseDir: str, nickname: str, domain: str, @@ -2501,7 +2569,19 @@ def createMediaTimeline(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'tlmedia', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, True, - pageNumber) + 0, False, 0, pageNumber) + + +def createNewsTimeline(session, baseDir: str, nickname: str, domain: str, + port: int, httpPrefix: str, itemsPerPage: int, + headerOnly: bool, newswireVotesThreshold: int, + positiveVoting: bool, votingTimeMins: int, + pageNumber=None) -> {}: + return createBoxIndexed({}, session, baseDir, 'outbox', 'news', + domain, port, httpPrefix, + itemsPerPage, headerOnly, True, + newswireVotesThreshold, positiveVoting, + votingTimeMins, pageNumber) def createOutbox(session, baseDir: str, nickname: str, domain: str, @@ -2511,7 +2591,7 @@ def createOutbox(session, baseDir: str, nickname: str, domain: str, return createBoxIndexed({}, session, baseDir, 'outbox', nickname, domain, port, httpPrefix, itemsPerPage, headerOnly, authorized, - pageNumber) + 0, False, 0, pageNumber) def createModeration(baseDir: str, nickname: str, domain: str, port: int, @@ -2775,7 +2855,7 @@ def addPostStringToTimeline(postStr: str, boxname: str, elif boxname == 'tlreplies': if boxActor not in postStr: return False - elif boxname == 'tlblogs': + elif boxname == 'tlblogs' or boxname == 'tlnews': if '"Create"' not in postStr: return False if '"Article"' not in postStr: @@ -2804,7 +2884,8 @@ def createBoxIndexed(recentPostsCache: {}, session, baseDir: str, boxname: str, nickname: str, domain: str, port: int, httpPrefix: str, itemsPerPage: int, headerOnly: bool, authorized: bool, - pageNumber=None) -> {}: + newswireVotesThreshold: int, positiveVoting: bool, + votingTimeMins: int, pageNumber=None) -> {}: """Constructs the box feed for a person with the given nickname """ if not authorized or not pageNumber: @@ -2812,7 +2893,7 @@ def createBoxIndexed(recentPostsCache: {}, if boxname != 'inbox' and boxname != 'dm' and \ boxname != 'tlreplies' and boxname != 'tlmedia' and \ - boxname != 'tlblogs' and \ + boxname != 'tlblogs' and boxname != 'tlnews' and \ boxname != 'outbox' and boxname != 'tlbookmarks' and \ boxname != 'bookmarks' and \ boxname != 'tlevents': @@ -2873,6 +2954,44 @@ def createBoxIndexed(recentPostsCache: {}, if not postFilename: break + # apply votes within this timeline + if newswireVotesThreshold > 0: + # note that the presence of an arrival file also indicates + # that this post is moderated + arrivalDate = \ + locateNewsArrival(baseDir, domain, postFilename) + if arrivalDate: + # how long has elapsed since this post arrived? + currDate = datetime.datetime.utcnow() + timeDiffMins = \ + int((currDate - arrivalDate).total_seconds() / 60) + # has the voting time elapsed? + if timeDiffMins < votingTimeMins: + # voting is still happening, so don't add this + # post to the timeline + continue + # if there a votes file for this post? + votesFilename = \ + locateNewsVotes(baseDir, domain, postFilename) + if votesFilename: + # load the votes file and count the votes + votesJson = loadJson(votesFilename, 0, 2) + if votesJson: + if not positiveVoting: + if votesOnNewswireItem(votesJson) >= \ + newswireVotesThreshold: + # Too many veto votes. + # Continue without incrementing + # the posts counter + continue + else: + if votesOnNewswireItem < \ + newswireVotesThreshold: + # Not enough votes. + # Continue without incrementing + # the posts counter + continue + # Skip through any posts previous to the current page if postsCtr < int((pageNumber - 1) * itemsPerPage): postsCtr += 1 diff --git a/pyjsonld.py b/pyjsonld.py new file mode 100644 index 000000000..a6da7f28f --- /dev/null +++ b/pyjsonld.py @@ -0,0 +1,4909 @@ +""" +Python implementation of JSON-LD processor + +This implementation is ported from the JavaScript implementation of +JSON-LD. + +.. module:: jsonld + :synopsis: Python implementation of JSON-LD + +.. moduleauthor:: Dave Longley +.. moduleauthor:: Mike Johnson +.. moduleauthor:: Tim McNamara +""" + +__copyright__ = 'Copyright (c) 2011-2014 Digital Bazaar, Inc.' +__license__ = 'New BSD license' +__version__ = '0.6.8' + +__all__ = [ + 'compact', 'expand', 'flatten', 'frame', 'link', 'from_rdf', 'to_rdf', + 'normalize', 'set_document_loader', 'get_document_loader', + 'parse_link_header', 'load_document', + 'register_rdf_parser', 'unregister_rdf_parser', + 'JsonLdProcessor', 'JsonLdError', 'ActiveContextCache'] + +import copy +import gzip +import hashlib +import io +import json +import os +import posixpath +import re +import socket +import ssl +import string +import sys +import traceback +from collections import deque, namedtuple +from contextlib import closing +from numbers import Integral, Real + +try: + from functools import cmp_to_key +except ImportError: + def cmp_to_key(mycmp): + """ + Convert a cmp= function into a key= function + + Source: http://hg.python.org/cpython/file/default/Lib/functools.py + """ + class K(object): + __slots__ = ['obj'] + + def __init__(self, obj): + self.obj = obj + + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + + def __ne__(self, other): + return mycmp(self.obj, other.obj) != 0 + __hash__ = None + return K + +# support python 2 +if sys.version_info[0] >= 3: + from urllib.request import build_opener as urllib_build_opener + from urllib.request import HTTPSHandler + import urllib.parse as urllib_parse + from http.client import HTTPSConnection + basestring = str + + def cmp(a, b): + return (a > b) - (a < b) +else: + from urllib2 import build_opener as urllib_build_opener + from urllib2 import HTTPSHandler + import urlparse as urllib_parse + from httplib import HTTPSConnection + +# XSD constants +XSD_BOOLEAN = 'http://www.w3.org/2001/XMLSchema#boolean' +XSD_DOUBLE = 'http://www.w3.org/2001/XMLSchema#double' +XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer' +XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string' + +# RDF constants +RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' +RDF_LIST = RDF + 'List' +RDF_FIRST = RDF + 'first' +RDF_REST = RDF + 'rest' +RDF_NIL = RDF + 'nil' +RDF_TYPE = RDF + 'type' +RDF_LANGSTRING = RDF + 'langString' + +# JSON-LD keywords +KEYWORDS = [ + '@base', + '@context', + '@container', + '@default', + '@embed', + '@explicit', + '@graph', + '@id', + '@index', + '@language', + '@list', + '@omitDefault', + '@preserve', + '@requireAll', + '@reverse', + '@set', + '@type', + '@value', + '@vocab'] + +# JSON-LD link header rel +LINK_HEADER_REL = 'http://www.w3.org/ns/json-ld#context' + +# Restraints +MAX_CONTEXT_URLS = 10 + + +def compact(input_, ctx, options=None): + """ + Performs JSON-LD compaction. + + :param input_: the JSON-LD input to compact. + :param ctx: the JSON-LD context to compact with. + :param [options]: the options to use. + [base] the base IRI to use. + [compactArrays] True to compact arrays to single values when + appropriate, False not to (default: True). + [graph] True to always output a top-level graph (default: False). + [expandContext] a context to expand with. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the compacted JSON-LD output. + """ + return JsonLdProcessor().compact(input_, ctx, options) + + +def expand(input_, options=None): + """ + Performs JSON-LD expansion. + + :param input_: the JSON-LD input to expand. + :param [options]: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the expanded JSON-LD output. + """ + return JsonLdProcessor().expand(input_, options) + + +def flatten(input_, ctx=None, options=None): + """ + Performs JSON-LD flattening. + + :param input_: the JSON-LD input to flatten. + :param ctx: the JSON-LD context to compact with (default: None). + :param [options]: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the flattened JSON-LD output. + """ + return JsonLdProcessor().flatten(input_, ctx, options) + + +def frame(input_, frame, options=None): + """ + Performs JSON-LD framing. + + :param input_: the JSON-LD input to frame. + :param frame: the JSON-LD frame to use. + :param [options]: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [embed] default @embed flag (default: True). + [explicit] default @explicit flag (default: False). + [requireAll] default @requireAll flag (default: True). + [omitDefault] default @omitDefault flag (default: False). + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the framed JSON-LD output. + """ + return JsonLdProcessor().frame(input_, frame, options) + + +def link(input_, ctx, options=None): + """ + **Experimental** + + Links a JSON-LD document's nodes in memory. + + :param input_: the JSON-LD document to link. + :param ctx: the JSON-LD context to apply or None. + :param [options]: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the linked JSON-LD output. + """ + # API matches running frame with a wildcard frame and embed: '@link' + # get arguments + frame = {'@embed': '@link'} + if ctx: + frame['@context'] = ctx + frame['@embed'] = '@link' + return frame(input, frame, options) + + +def normalize(input_, options=None): + """ + Performs JSON-LD normalization. + + :param input_: the JSON-LD input to normalize. + :param [options]: the options to use. + [base] the base IRI to use. + [format] the format if output is a string: + 'application/nquads' for N-Quads. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the normalized JSON-LD output. + """ + return JsonLdProcessor().normalize(input_, options) + + +def from_rdf(input_, options=None): + """ + Converts an RDF dataset to JSON-LD. + + :param input_: a serialized string of RDF in a format specified + by the format option or an RDF dataset to convert. + :param [options]: the options to use: + [format] the format if input is a string: + 'application/nquads' for N-Quads (default: 'application/nquads'). + [useRdfType] True to use rdf:type, False to use @type (default: False). + [useNativeTypes] True to convert XSD types into native types + (boolean, integer, double), False not to (default: True). + + :return: the JSON-LD output. + """ + return JsonLdProcessor().from_rdf(input_, options) + + +def to_rdf(input_, options=None): + """ + Outputs the RDF dataset found in the given JSON-LD object. + + :param input_: the JSON-LD input. + :param [options]: the options to use. + [base] the base IRI to use. + [format] the format to use to output a string: + 'application/nquads' for N-Quads. + [produceGeneralizedRdf] true to output generalized RDF, false + to produce only standard RDF (default: false). + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the resulting RDF dataset (or a serialization of it). + """ + return JsonLdProcessor().to_rdf(input_, options) + + +def set_document_loader(load_document): + """ + Sets the default JSON-LD document loader. + + :param load_document(url): the document loader to use. + """ + global _default_document_loader + _default_document_loader = load_document + + +def get_document_loader(): + """ + Gets the default JSON-LD document loader. + + :return: the default document loader. + """ + return _default_document_loader + + +def parse_link_header(header): + """ + Parses a link header. The results will be key'd by the value of "rel". + + Link: ; \ + rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + + Parses as: { + 'http://www.w3.org/ns/json-ld#context': { + target: http://json-ld.org/contexts/person.jsonld, + type: 'application/ld+json' + } + } + + If there is more than one "rel" with the same IRI, then entries in the + resulting map for that "rel" will be lists. + + :param header: the link header to parse. + + :return: the parsed result. + """ + rval = {} + # split on unbracketed/unquoted commas + entries = re.findall(r'(?:<[^>]*?>|"[^"]*?"|[^,])+', header) + if not entries: + return rval + r_link_header = r'\s*<([^>]*?)>\s*(?:;\s*(.*))?' + for entry in entries: + match = re.search(r_link_header, entry) + if not match: + continue + match = match.groups() + result = {'target': match[0]} + params = match[1] + r_params = r'(.*?)=(?:(?:"([^"]*?)")|([^"]*?))\s*(?:(?:;\s*)|$)' + matches = re.findall(r_params, params) + for match in matches: + result[match[0]] = match[2] if match[1] is None else match[1] + rel = result.get('rel', '') + if isinstance(rval.get(rel), list): + rval[rel].append(result) + elif rel in rval: + rval[rel] = [rval[rel], result] + else: + rval[rel] = result + return rval + + +def load_document(url): + """ + Retrieves JSON-LD at the given URL. + + :param url: the URL to retrieve. + + :return: the RemoteDocument. + """ + try: + # validate URL + pieces = urllib_parse.urlparse(url) + if (not all([pieces.scheme, pieces.netloc]) or + pieces.scheme not in ['http', 'https'] or + set(pieces.netloc) > set( + string.ascii_letters + string.digits + '-.:')): + raise JsonLdError( + 'URL could not be dereferenced; only "http" and "https" ' + 'URLs are supported.', + 'jsonld.InvalidUrl', {'url': url}, + code='loading document failed') + https_handler = VerifiedHTTPSHandler() + url_opener = urllib_build_opener(https_handler) + url_opener.addheaders = [ + ('Accept', 'application/ld+json, application/json'), + ('Accept-Encoding', 'deflate')] + with closing(url_opener.open(url)) as handle: + if handle.info().get('Content-Encoding') == 'gzip': + buf = io.BytesIO(handle.read()) + f = gzip.GzipFile(fileobj=buf, mode='rb') + data = f.read() + else: + data = handle.read() + doc = { + 'contextUrl': None, + 'documentUrl': url, + 'document': data.decode('utf8') + } + doc['documentUrl'] = handle.geturl() + headers = dict(handle.info()) + content_type = headers.get('content-type') + link_header = headers.get('link') + if link_header and content_type != 'application/ld+json': + link_header = parse_link_header(link_header).get( + LINK_HEADER_REL) + # only 1 related link header permitted + if isinstance(link_header, list): + raise JsonLdError( + 'URL could not be dereferenced, it has more than one ' + 'associated HTTP Link Header.', + 'jsonld.LoadDocumentError', + {'url': url}, + code='multiple context link headers') + if link_header: + doc['contextUrl'] = link_header['target'] + return doc + except JsonLdError as e: + raise e + except Exception as cause: + raise JsonLdError( + 'Could not retrieve a JSON-LD document from the URL.', + 'jsonld.LoadDocumentError', code='loading document failed', + cause=cause) + + +def register_rdf_parser(content_type, parser): + """ + Registers a global RDF parser by content-type, for use with + from_rdf. Global parsers will be used by JsonLdProcessors that + do not register their own parsers. + + :param content_type: the content-type for the parser. + :param parser(input): the parser function (takes a string as + a parameter and returns an RDF dataset). + """ + global _rdf_parsers + _rdf_parsers[content_type] = parser + + +def unregister_rdf_parser(content_type): + """ + Unregisters a global RDF parser by content-type. + + :param content_type: the content-type for the parser. + """ + global _rdf_parsers + if content_type in _rdf_parsers: + del _rdf_parsers[content_type] + + +def prepend_base(base, iri): + """ + Prepends a base IRI to the given relative IRI. + + :param base: the base IRI. + :param iri: the relative IRI. + + :return: the absolute IRI. + """ + # skip IRI processing + if base is None: + return iri + + # already an absolute iri + if _is_absolute_iri(iri): + return iri + + # parse IRIs + base = parse_url(base) + rel = parse_url(iri) + + # per RFC3986 5.2.2 + transform = { + 'scheme': base.scheme + } + + if rel.authority is not None: + transform['authority'] = rel.authority + transform['path'] = rel.path + transform['query'] = rel.query + else: + transform['authority'] = base.authority + + if rel.path == '': + transform['path'] = base.path + if rel.query is not None: + transform['query'] = rel.query + else: + transform['query'] = base.query + else: + if rel.path.startswith('/'): + # IRI represents an absolute path + transform['path'] = rel.path + else: + # merge paths + path = base.path + + # append relative path to the end of the last + # directory from base + if rel.path != '': + path = path[0:path.rfind('/') + 1] + if len(path) > 0 and not path.endswith('/'): + path += '/' + path += rel.path + + transform['path'] = path + + transform['query'] = rel.query + + # normalize path + path = transform['path'] + add_slash = path.endswith('/') + path = posixpath.normpath(path) + if not path.endswith('/') and add_slash: + path += '/' + # do not include '.' path + if path == '.': + path = '' + transform['path'] = path + + transform['fragment'] = rel.fragment + + # construct URL + rval = unparse_url(transform) + + # handle empty base case + if rval == '': + rval = './' + + return rval + + +def remove_base(base, iri): + """ + Removes a base IRI from the given absolute IRI. + + :param base: the base IRI. + :param iri: the absolute IRI. + + :return: the relative IRI if relative to base, otherwise the absolute IRI. + """ + # skip IRI processing + if base is None: + return iri + + base = parse_url(base) + rel = parse_url(iri) + + # schemes and network locations (authorities) don't match, don't alter IRI + if not (base.scheme == rel.scheme and base.authority == rel.authority): + return iri + + path = posixpath.relpath(rel.path, base.path) if rel.path else '' + path = posixpath.normpath(path) + # workaround a relpath bug in Python 2.6 (http://bugs.python.org/issue5117) + if base.path == '/' and path.startswith('../'): + path = path[3:] + if path == '.' and not rel.path.endswith('/') and not ( + rel.query or rel.fragment): + path = posixpath.basename(rel.path) + if rel.path.endswith('/') and not path.endswith('/'): + path += '/' + + # adjustments for base that is not a directory + if not base.path.endswith('/'): + if path.startswith('../'): + path = path[3:] + elif path.startswith('./'): + path = path[2:] + elif path.startswith('.'): + path = path[1:] + + return unparse_url((None, None, path, rel.query, rel.fragment)) or './' + + +ParsedUrl = namedtuple( + 'ParsedUrl', ['scheme', 'authority', 'path', 'query', 'fragment']) + + +def parse_url(url): + # regex from RFC 3986 + p = r'^(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?' + m = re.match(p, url) + return ParsedUrl(*m.groups()) + + +def unparse_url(parsed): + if isinstance(parsed, dict): + parsed = ParsedUrl(**parsed) + elif isinstance(parsed, list) or isinstance(parsed, tuple): + parsed = ParsedUrl(*parsed) + rval = '' + if parsed.scheme: + rval += parsed.scheme + ':' + if parsed.authority is not None: + rval += '//' + parsed.authority + rval += parsed.path + if parsed.query is not None: + rval += '?' + parsed.query + if parsed.fragment is not None: + rval += '#' + parsed.fragment + return rval + + +# The default JSON-LD document loader. +_default_document_loader = load_document + +# Registered global RDF parsers hashed by content-type. +_rdf_parsers = {} + + +class JsonLdProcessor(object): + """ + A JSON-LD processor. + """ + + def __init__(self): + """ + Initialize the JSON-LD processor. + """ + # processor-specific RDF parsers + self.rdf_parsers = None + + def compact(self, input_, ctx, options): + """ + Performs JSON-LD compaction. + + :param input_: the JSON-LD input to compact. + :param ctx: the context to compact with. + :param options: the options to use. + [base] the base IRI to use. + [compactArrays] True to compact arrays to single values when + appropriate, False not to (default: True). + [graph] True to always output a top-level graph (default: False). + [expandContext] a context to expand with. + [skipExpansion] True to assume the input is expanded and skip + expansion, False not to, (default: False). + [activeCtx] True to also return the active context used. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the compacted JSON-LD output. + """ + if ctx is None: + raise JsonLdError( + 'The compaction context must not be null.', + 'jsonld.CompactError', code='invalid local context') + + # nothing to compact + if input_ is None: + return None + + # set default options + options = options or {} + options.setdefault('base', input_ if _is_string(input_) else '') + options.setdefault('compactArrays', True) + options.setdefault('graph', False) + options.setdefault('skipExpansion', False) + options.setdefault('activeCtx', False) + options.setdefault('documentLoader', _default_document_loader) + options.setdefault('link', False) + if options['link']: + # force skip expansion when linking, "link" is not part of the + # public API, it should only be called from framing + options['skipExpansion'] = True + + if options['skipExpansion']: + expanded = input_ + else: + # expand input + try: + expanded = self.expand(input_, options) + except JsonLdError as cause: + raise JsonLdError( + 'Could not expand input before compaction.', + 'jsonld.CompactError', cause=cause) + + # process context + active_ctx = self._get_initial_context(options) + try: + active_ctx = self.process_context(active_ctx, ctx, options) + except JsonLdError as cause: + raise JsonLdError( + 'Could not process context before compaction.', + 'jsonld.CompactError', cause=cause) + + # do compaction + compacted = self._compact(active_ctx, None, expanded, options) + + if (options['compactArrays'] and not options['graph'] and + _is_array(compacted)): + # simplify to a single item + if len(compacted) == 1: + compacted = compacted[0] + # simplify to an empty object + elif len(compacted) == 0: + compacted = {} + # always use an array if graph options is on + elif options['graph']: + compacted = JsonLdProcessor.arrayify(compacted) + + # follow @context key + if _is_object(ctx) and '@context' in ctx: + ctx = ctx['@context'] + + # build output context + ctx = copy.deepcopy(ctx) + ctx = JsonLdProcessor.arrayify(ctx) + + # remove empty contexts + tmp = ctx + ctx = [] + for v in tmp: + if not _is_object(v) or len(v) > 0: + ctx.append(v) + + # remove array if only one context + ctx_length = len(ctx) + has_context = (ctx_length > 0) + if ctx_length == 1: + ctx = ctx[0] + + # add context and/or @graph + if _is_array(compacted): + # use '@graph' keyword + kwgraph = self._compact_iri(active_ctx, '@graph') + graph = compacted + compacted = {} + if has_context: + compacted['@context'] = ctx + compacted[kwgraph] = graph + elif _is_object(compacted) and has_context: + # reorder keys so @context is first + graph = compacted + compacted = {} + compacted['@context'] = ctx + for k, v in graph.items(): + compacted[k] = v + + if options['activeCtx']: + return {'compacted': compacted, 'activeCtx': active_ctx} + else: + return compacted + + def expand(self, input_, options): + """ + Performs JSON-LD expansion. + + :param input_: the JSON-LD input to expand. + :param options: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [keepFreeFloatingNodes] True to keep free-floating nodes, + False not to (default: False). + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the expanded JSON-LD output. + """ + # set default options + options = options or {} + options.setdefault('keepFreeFloatingNodes', False) + options.setdefault('documentLoader', _default_document_loader) + + # if input is a string, attempt to dereference remote document + if _is_string(input_): + remote_doc = options['documentLoader'](input_) + else: + remote_doc = { + 'contextUrl': None, + 'documentUrl': None, + 'document': input_ + } + + try: + if remote_doc['document'] is None: + raise JsonLdError( + 'No remote document found at the given URL.', + 'jsonld.NullRemoteDocument') + if _is_string(remote_doc['document']): + remote_doc['document'] = json.loads(remote_doc['document']) + except Exception as cause: + raise JsonLdError( + 'Could not retrieve a JSON-LD document from the URL.', + 'jsonld.LoadDocumentError', + {'remoteDoc': remote_doc}, code='loading document failed', + cause=cause) + + # set default base + options.setdefault('base', remote_doc['documentUrl'] or '') + + # build meta-object and retrieve all @context urls + input_ = { + 'document': copy.deepcopy(remote_doc['document']), + 'remoteContext': {'@context': remote_doc['contextUrl']} + } + if 'expandContext' in options: + expand_context = copy.deepcopy(options['expandContext']) + if _is_object(expand_context) and '@context' in expand_context: + input_['expandContext'] = expand_context + else: + input_['expandContext'] = {'@context': expand_context} + + try: + self._retrieve_context_urls( + input_, {}, options['documentLoader'], options['base']) + except Exception as cause: + raise JsonLdError( + 'Could not perform JSON-LD expansion.', + 'jsonld.ExpandError', cause=cause) + + active_ctx = self._get_initial_context(options) + document = input_['document'] + remote_context = input_['remoteContext']['@context'] + + # process optional expandContext + if 'expandContext' in input_: + active_ctx = self.process_context( + active_ctx, input_['expandContext']['@context'], options) + + # process remote context from HTTP Link Header + if remote_context is not None: + active_ctx = self.process_context( + active_ctx, remote_context, options) + + # do expansion + expanded = self._expand(active_ctx, None, document, options, False) + + # optimize away @graph with no other properties + if (_is_object(expanded) and '@graph' in expanded and + len(expanded) == 1): + expanded = expanded['@graph'] + elif expanded is None: + expanded = [] + + # normalize to an array + return JsonLdProcessor.arrayify(expanded) + + def flatten(self, input_, ctx, options): + """ + Performs JSON-LD flattening. + + :param input_: the JSON-LD input to flatten. + :param ctx: the JSON-LD context to compact with (default: None). + :param options: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the flattened JSON-LD output. + """ + options = options or {} + options.setdefault('base', input_ if _is_string(input_) else '') + options.setdefault('documentLoader', _default_document_loader) + + try: + # expand input + expanded = self.expand(input_, options) + except Exception as cause: + raise JsonLdError( + 'Could not expand input before flattening.', + 'jsonld.FlattenError', cause=cause) + + # do flattening + flattened = self._flatten(expanded) + + if ctx is None: + return flattened + + # compact result (force @graph option to true, skip expansion) + options['graph'] = True + options['skipExpansion'] = True + try: + compacted = self.compact(flattened, ctx, options) + except Exception as cause: + raise JsonLdError( + 'Could not compact flattened output.', + 'jsonld.FlattenError', cause=cause) + + return compacted + + def frame(self, input_, frame, options): + """ + Performs JSON-LD framing. + + :param input_: the JSON-LD object to frame. + :param frame: the JSON-LD frame to use. + :param options: the options to use. + [base] the base IRI to use. + [expandContext] a context to expand with. + [embed] default @embed flag: '@last', '@always', '@never', '@link' + (default: '@last'). + [explicit] default @explicit flag (default: False). + [requireAll] default @requireAll flag (default: True). + [omitDefault] default @omitDefault flag (default: False). + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the framed JSON-LD output. + """ + # set default options + options = options or {} + options.setdefault('base', input_ if _is_string(input_) else '') + options.setdefault('compactArrays', True) + options.setdefault('embed', '@last') + options.setdefault('explicit', False) + options.setdefault('requireAll', True) + options.setdefault('omitDefault', False) + options.setdefault('documentLoader', _default_document_loader) + + # if frame is a string, attempt to dereference remote document + if _is_string(frame): + remote_frame = options['documentLoader'](frame) + else: + remote_frame = { + 'contextUrl': None, + 'documentUrl': None, + 'document': frame + } + + try: + if remote_frame['document'] is None: + raise JsonLdError( + 'No remote document found at the given URL.', + 'jsonld.NullRemoteDocument') + if _is_string(remote_frame['document']): + remote_frame['document'] = json.loads(remote_frame['document']) + except Exception as cause: + raise JsonLdError( + 'Could not retrieve a JSON-LD document from the URL.', + 'jsonld.LoadDocumentError', + {'remoteDoc': remote_frame}, code='loading document failed', + cause=cause) + + # preserve frame context + frame = remote_frame['document'] + if frame is not None: + ctx = frame.get('@context', {}) + if remote_frame['contextUrl'] is not None: + if ctx is not None: + ctx = remote_frame['contextUrl'] + else: + ctx = JsonLdProcessor.arrayify(ctx) + ctx.append(remote_frame['contextUrl']) + frame['@context'] = ctx + + try: + # expand input + expanded = self.expand(input_, options) + except JsonLdError as cause: + raise JsonLdError( + 'Could not expand input before framing.', + 'jsonld.FrameError', cause=cause) + + try: + # expand frame + opts = copy.deepcopy(options) + opts['keepFreeFloatingNodes'] = True + expanded_frame = self.expand(frame, opts) + except JsonLdError as cause: + raise JsonLdError( + 'Could not expand frame before framing.', + 'jsonld.FrameError', cause=cause) + + # do framing + framed = self._frame(expanded, expanded_frame, options) + + try: + # compact result (force @graph option to True, skip expansion, + # check for linked embeds) + options['graph'] = True + options['skipExpansion'] = True + options['link'] = {} + options['activeCtx'] = True + result = self.compact(framed, ctx, options) + except JsonLdError as cause: + raise JsonLdError( + 'Could not compact framed output.', + 'jsonld.FrameError', cause=cause) + + compacted = result['compacted'] + active_ctx = result['activeCtx'] + + # get graph alias + graph = self._compact_iri(active_ctx, '@graph') + # remove @preserve from results + compacted[graph] = self._remove_preserve( + active_ctx, compacted[graph], options) + return compacted + + def normalize(self, input_, options): + """ + Performs RDF normalization on the given JSON-LD input. + + :param input_: the JSON-LD input to normalize. + :param options: the options to use. + [base] the base IRI to use. + [format] the format if output is a string: + 'application/nquads' for N-Quads. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the normalized output. + """ + # set default options + options = options or {} + options.setdefault('base', input_ if _is_string(input_) else '') + options.setdefault('documentLoader', _default_document_loader) + + try: + # convert to RDF dataset then do normalization + opts = copy.deepcopy(options) + if 'format' in opts: + del opts['format'] + opts['produceGeneralizedRdf'] = False + dataset = self.to_rdf(input_, opts) + except JsonLdError as cause: + raise JsonLdError( + 'Could not convert input to RDF dataset before normalization.', + 'jsonld.NormalizeError', cause=cause) + + # do normalization + return self._normalize(dataset, options) + + def from_rdf(self, dataset, options): + """ + Converts an RDF dataset to JSON-LD. + + :param dataset: a serialized string of RDF in a format specified by + the format option or an RDF dataset to convert. + :param options: the options to use. + [format] the format if input is a string: + 'application/nquads' for N-Quads (default: 'application/nquads'). + [useRdfType] True to use rdf:type, False to use @type + (default: False). + [useNativeTypes] True to convert XSD types into native types + (boolean, integer, double), False not to (default: False). + + :return: the JSON-LD output. + """ + global _rdf_parsers + + # set default options + options = options or {} + options.setdefault('useRdfType', False) + options.setdefault('useNativeTypes', False) + + if ('format' not in options) and _is_string(dataset): + options['format'] = 'application/nquads' + + # handle special format + if 'format' in options: + # supported formats (processor-specific and global) + if ((self.rdf_parsers is not None and + not options['format'] in self.rdf_parsers) or + (self.rdf_parsers is None and + not options['format'] in _rdf_parsers)): + raise JsonLdError( + 'Unknown input format.', + 'jsonld.UnknownFormat', {'format': options['format']}) + + if self.rdf_parsers is not None: + parser = self.rdf_parsers[options['format']] + else: + parser = _rdf_parsers[options['format']] + dataset = parser(dataset) + + # convert from RDF + return self._from_rdf(dataset, options) + + def to_rdf(self, input_, options): + """ + Outputs the RDF dataset found in the given JSON-LD object. + + :param input_: the JSON-LD input. + :param options: the options to use. + [base] the base IRI to use. + [format] the format if input is a string: + 'application/nquads' for N-Quads. + [produceGeneralizedRdf] true to output generalized RDF, false + to produce only standard RDF (default: false). + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the resulting RDF dataset (or a serialization of it). + """ + # set default options + options = options or {} + options.setdefault('base', input_ if _is_string(input_) else '') + options.setdefault('produceGeneralizedRdf', False) + options.setdefault('documentLoader', _default_document_loader) + + try: + # expand input + expanded = self.expand(input_, options) + except JsonLdError as cause: + raise JsonLdError( + 'Could not expand input before serialization to ' + 'RDF.', 'jsonld.RdfError', cause=cause) + + # create node map for default graph (and any named graphs) + namer = UniqueNamer('_:b') + node_map = {'@default': {}} + self._create_node_map(expanded, node_map, '@default', namer) + + # output RDF dataset + dataset = {} + for graph_name, graph in sorted(node_map.items()): + # skip relative IRIs + if graph_name == '@default' or _is_absolute_iri(graph_name): + dataset[graph_name] = self._graph_to_rdf(graph, namer, options) + + # convert to output format + if 'format' in options: + if options['format'] == 'application/nquads': + return self.to_nquads(dataset) + raise JsonLdError( + 'Unknown output format.', + 'jsonld.UnknownFormat', {'format': options['format']}) + return dataset + + def process_context(self, active_ctx, local_ctx, options): + """ + Processes a local context, retrieving any URLs as necessary, and + returns a new active context in its callback. + + :param active_ctx: the current active context. + :param local_ctx: the local context to process. + :param options: the options to use. + [documentLoader(url)] the document loader + (default: _default_document_loader). + + :return: the new active context. + """ + # return initial context early for None context + if local_ctx is None: + return self._get_initial_context(options) + + # set default options + options = options or {} + options.setdefault('base', '') + options.setdefault('documentLoader', _default_document_loader) + + # retrieve URLs in local_ctx + local_ctx = copy.deepcopy(local_ctx) + if (_is_string(local_ctx) or ( + _is_object(local_ctx) and '@context' not in local_ctx)): + local_ctx = {'@context': local_ctx} + try: + self._retrieve_context_urls( + local_ctx, {}, options['documentLoader'], options['base']) + except Exception as cause: + raise JsonLdError( + 'Could not process JSON-LD context.', + 'jsonld.ContextError', cause=cause) + + # process context + return self._process_context(active_ctx, local_ctx, options) + + def register_rdf_parser(self, content_type, parser): + """ + Registers a processor-specific RDF parser by content-type. + Global parsers will no longer be used by this processor. + + :param content_type: the content-type for the parser. + :param parser(input): the parser function (takes a string as + a parameter and returns an RDF dataset). + """ + if self.rdf_parsers is None: + self.rdf_parsers = {} + self.rdf_parsers[content_type] = parser + + def unregister_rdf_parser(self, content_type): + """ + Unregisters a process-specific RDF parser by content-type. + If there are no remaining processor-specific parsers, then the global + parsers will be re-enabled. + + :param content_type: the content-type for the parser. + """ + if (self.rdf_parsers is not None and + content_type in self.rdf_parsers): + del self.rdf_parsers[content_type] + if len(self.rdf_parsers) == 0: + self.rdf_parsers = None + + @staticmethod + def has_property(subject, property): + """ + Returns True if the given subject has the given property. + + :param subject: the subject to check. + :param property: the property to look for. + + :return: True if the subject has the given property, False if not. + """ + if property in subject: + value = subject[property] + return not _is_array(value) or len(value) > 0 + return False + + @staticmethod + def has_value(subject, property, value): + """ + Determines if the given value is a property of the given subject. + + :param subject: the subject to check. + :param property: the property to check. + :param value: the value to check. + + :return: True if the value exists, False if not. + """ + if JsonLdProcessor.has_property(subject, property): + val = subject[property] + is_list = _is_list(val) + if _is_array(val) or is_list: + if is_list: + val = val['@list'] + for v in val: + if JsonLdProcessor.compare_values(value, v): + return True + # avoid matching the set of values with an array value parameter + elif not _is_array(value): + return JsonLdProcessor.compare_values(value, val) + return False + + @staticmethod + def add_value(subject, property, value, options={}): + """ + Adds a value to a subject. If the value is an array, all values in the + array will be added. + + :param subject: the subject to add the value to. + :param property: the property that relates the value to the subject. + :param value: the value to add. + :param [options]: the options to use: + [propertyIsArray] True if the property is always + an array, False if not (default: False). + [allowDuplicate] True to allow duplicates, False not to (uses + a simple shallow comparison of subject ID or value) + (default: True). + """ + options.setdefault('propertyIsArray', False) + options.setdefault('allowDuplicate', True) + + if _is_array(value): + if (len(value) == 0 and options['propertyIsArray'] and + property not in subject): + subject[property] = [] + for v in value: + JsonLdProcessor.add_value(subject, property, v, options) + elif property in subject: + # check if subject already has value if duplicates not allowed + has_value = \ + (not options['allowDuplicate'] and + JsonLdProcessor.has_value(subject, property, value)) + + # make property an array if value not present or always an array + if (not _is_array(subject[property]) and + (not has_value or options['propertyIsArray'])): + subject[property] = [subject[property]] + + # add new value + if not has_value: + subject[property].append(value) + else: + # add new value as set or single value + subject[property] = ( + [value] if options['propertyIsArray'] else value) + + @staticmethod + def get_values(subject, property): + """ + Gets all of the values for a subject's property as an array. + + :param subject: the subject. + :param property: the property. + + :return: all of the values for a subject's property as an array. + """ + return JsonLdProcessor.arrayify(subject.get(property) or []) + + @staticmethod + def remove_property(subject, property): + """ + Removes a property from a subject. + + :param subject: the subject. + :param property: the property. + """ + del subject[property] + + @staticmethod + def remove_value(subject, property, value, options={}): + """ + Removes a value from a subject. + + :param subject: the subject. + :param property: the property that relates the value to the subject. + :param value: the value to remove. + :param [options]: the options to use: + [propertyIsArray]: True if the property is always an array, + False if not (default: False). + """ + options.setdefault('propertyIsArray', False) + + # filter out value + def filter_value(e): + return not JsonLdProcessor.compare_values(e, value) + values = JsonLdProcessor.get_values(subject, property) + values = list(filter(filter_value, values)) + + if len(values) == 0: + JsonLdProcessor.remove_property(subject, property) + elif len(values) == 1 and not options['propertyIsArray']: + subject[property] = values[0] + else: + subject[property] = values + + @staticmethod + def compare_values(v1, v2): + """ + Compares two JSON-LD values for equality. Two JSON-LD values will be + considered equal if: + + 1. They are both primitives of the same type and value. + 2. They are both @values with the same @value, @type, @language, + and @index, OR + 3. They both have @ids that are the same. + + :param v1: the first value. + :param v2: the second value. + + :return: True if v1 and v2 are considered equal, False if not. + """ + # 1. equal primitives + if not _is_object(v1) and not _is_object(v2) and v1 == v2: + type1 = type(v1) + type2 = type(v2) + if type1 == bool or type2 == bool: + return type1 == type2 + return True + + # 2. equal @values + if (_is_value(v1) and _is_value(v2) and + v1['@value'] == v2['@value'] and + v1.get('@type') == v2.get('@type') and + v1.get('@language') == v2.get('@language') and + v1.get('@index') == v2.get('@index')): + type1 = type(v1['@value']) + type2 = type(v2['@value']) + if type1 == bool or type2 == bool: + return type1 == type2 + return True + + # 3. equal @ids + if (_is_object(v1) and '@id' in v1 and + _is_object(v2) and '@id' in v2): + return v1['@id'] == v2['@id'] + + return False + + @staticmethod + def get_context_value(ctx, key, type_): + """ + Gets the value for the given active context key and type, None if none + is set. + + :param ctx: the active context. + :param key: the context key. + :param [type_]: the type of value to get (eg: '@id', '@type'), if not + specified gets the entire entry for a key, None if not found. + + :return: mixed the value. + """ + rval = None + + # return None for invalid key + if key is None: + return rval + + # get default language + if type_ == '@language' and type_ in ctx: + rval = ctx[type_] + + # get specific entry information + if key in ctx['mappings']: + entry = ctx['mappings'][key] + if entry is None: + return None + + # return whole entry + if type_ is None: + rval = entry + # return entry value for type + elif type_ in entry: + rval = entry[type_] + + return rval + + @staticmethod + def parse_nquads(input_): + """ + Parses RDF in the form of N-Quads. + + :param input_: the N-Quads input to parse. + + :return: an RDF dataset. + """ + # define partial regexes + iri = '(?:<([^:]+:[^>]*)>)' + bnode = '(_:(?:[A-Za-z][A-Za-z0-9]*))' + plain = '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"' + datatype = '(?:\\^\\^' + iri + ')' + language = '(?:@([a-z]+(?:-[a-z0-9]+)*))' + literal = '(?:' + plain + '(?:' + datatype + '|' + language + ')?)' + ws = '[ \\t]+' + wso = '[ \\t]*' + eoln = r'(?:\r\n)|(?:\n)|(?:\r)' + empty = r'^' + wso + '$' + + # define quad part regexes + subject = '(?:' + iri + '|' + bnode + ')' + ws + property = iri + ws + object = '(?:' + iri + '|' + bnode + '|' + literal + ')' + wso + graph = '(?:\\.|(?:(?:' + iri + '|' + bnode + ')' + wso + '\\.))' + + # Note: Notice that the graph position does not include literals + # even though they are specified as a possible value in the + # N-Quads note (http://sw.deri.org/2008/07/n-quads/). This is + # intentional, as literals in that position are not supported by the + # RDF data model or the JSON-LD data model. + # See: https://github.com/digitalbazaar/pyld/pull/19 + + # full quad regex + quad = r'^' + wso + subject + property + object + graph + wso + '$' + + # build RDF dataset + dataset = {} + + # split N-Quad input into lines + lines = re.split(eoln, input_) + line_number = 0 + for line in lines: + line_number += 1 + + # skip empty lines + if re.search(empty, line) is not None: + continue + + # parse quad + match = re.search(quad, line) + if match is None: + raise JsonLdError( + 'Error while parsing N-Quads invalid quad.', + 'jsonld.ParseError', {'line': line_number}) + match = match.groups() + + # create RDF triple + triple = {'subject': {}, 'predicate': {}, 'object': {}} + + # get subject + if match[0] is not None: + triple['subject'] = {'type': 'IRI', 'value': match[0]} + else: + triple['subject'] = {'type': 'blank node', 'value': match[1]} + + # get predicate + triple['predicate'] = {'type': 'IRI', 'value': match[2]} + + # get object + if match[3] is not None: + triple['object'] = {'type': 'IRI', 'value': match[3]} + elif match[4] is not None: + triple['object'] = {'type': 'blank node', 'value': match[4]} + else: + triple['object'] = {'type': 'literal'} + replacements = { + '\\"': '\"', + '\\t': '\t', + '\\n': '\n', + '\\r': '\r', + '\\\\': '\\' + } + unescaped = match[5] + for match, repl in replacements.items(): + unescaped = unescaped.replace(match, repl) + if match[6] is not None: + triple['object']['datatype'] = match[6] + elif match[7] is not None: + triple['object']['datatype'] = RDF_LANGSTRING + triple['object']['language'] = match[7] + else: + triple['object']['datatype'] = XSD_STRING + triple['object']['value'] = unescaped + + # get graph name ('@default' is used for the default graph) + name = '@default' + if match[8] is not None: + name = match[8] + elif match[9] is not None: + name = match[9] + + # initialize graph in dataset + if name not in dataset: + dataset[name] = [triple] + # add triple if unique to its graph + else: + unique = True + triples = dataset[name] + for t in dataset[name]: + if JsonLdProcessor._compare_rdf_triples(t, triple): + unique = False + break + if unique: + triples.append(triple) + + return dataset + + @staticmethod + def to_nquads(dataset): + """ + Converts an RDF dataset to N-Quads. + + :param dataset: the RDF dataset to convert. + + :return: the N-Quads string. + """ + quads = [] + for graph_name, triples in dataset.items(): + for triple in triples: + if graph_name == '@default': + graph_name = None + quads.append(JsonLdProcessor.to_nquad(triple, graph_name)) + quads.sort() + return ''.join(quads) + + @staticmethod + def to_nquad(triple, graph_name, bnode=None): + """ + Converts an RDF triple and graph name to an N-Quad string (a single + quad). + + :param triple: the RDF triple to convert. + :param graph_name: the name of the graph containing the triple, None + for the default graph. + :param bnode: the bnode the quad is mapped to (optional, for + use during normalization only). + + :return: the N-Quad string. + """ + s = triple['subject'] + p = triple['predicate'] + o = triple['object'] + g = graph_name + + quad = '' + + # subject is an IRI + if s['type'] == 'IRI': + quad += '<' + s['value'] + '>' + # bnode normalization mode + elif bnode is not None: + quad += '_:a' if s['value'] == bnode else '_:z' + # bnode normal mode + else: + quad += s['value'] + quad += ' ' + + # property is an IRI + if p['type'] == 'IRI': + quad += '<' + p['value'] + '>' + # FIXME: TBD what to do with bnode predicates during normalization + # bnode normalization mode + elif bnode is not None: + quad += '_:p' + # bnode normal mode + else: + quad += p['value'] + quad += ' ' + + # object is IRI, bnode, or literal + if o['type'] == 'IRI': + quad += '<' + o['value'] + '>' + elif(o['type'] == 'blank node'): + # normalization mode + if bnode is not None: + quad += '_:a' if o['value'] == bnode else '_:z' + # normal mode + else: + quad += o['value'] + else: + replacements = { + '\\': '\\\\', + '\t': '\\t', + '\n': '\\n', + '\r': '\\r', + '\"': '\\"' + } + escaped = o['value'] + for match, repl in replacements.items(): + escaped = escaped.replace(match, repl) + quad += '"' + escaped + '"' + if o['datatype'] == RDF_LANGSTRING: + if o['language']: + quad += '@' + o['language'] + elif o['datatype'] != XSD_STRING: + quad += '^^<' + o['datatype'] + '>' + + # graph + if g is not None: + if not g.startswith('_:'): + quad += ' <' + g + '>' + elif bnode is not None: + quad += ' _:g' + else: + quad += ' ' + g + + quad += ' .\n' + return quad + + @staticmethod + def arrayify(value): + """ + If value is an array, returns value, otherwise returns an array + containing value as the only element. + + :param value: the value. + + :return: an array. + """ + return value if _is_array(value) else [value] + + @staticmethod + def _compare_rdf_triples(t1, t2): + """ + Compares two RDF triples for equality. + + :param t1: the first triple. + :param t2: the second triple. + + :return: True if the triples are the same, False if not. + """ + for attr in ['subject', 'predicate', 'object']: + if(t1[attr]['type'] != t2[attr]['type'] or + t1[attr]['value'] != t2[attr]['value']): + return False + + if t1['object'].get('language') != t2['object'].get('language'): + return False + if t1['object'].get('datatype') != t2['object'].get('datatype'): + return False + + return True + + def _compact(self, active_ctx, active_property, element, options): + """ + Recursively compacts an element using the given active context. All + values must be in expanded form before this method is called. + + :param active_ctx: the active context to use. + :param active_property: the compacted property with the element to + compact, None for none. + :param element: the element to compact. + :param options: the compaction options. + + :return: the compacted value. + """ + # recursively compact array + if _is_array(element): + rval = [] + for e in element: + # compact, dropping any None values + e = self._compact(active_ctx, active_property, e, options) + if e is not None: + rval.append(e) + if options['compactArrays'] and len(rval) == 1: + # use single element if no container is specified + container = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@container') + if container is None: + rval = rval[0] + return rval + + # recursively compact object + if _is_object(element): + if(options['link'] and '@id' in element and + element['@id'] in options['link']): + # check for a linked element to reuse + linked = options['link'][element['@id']] + for link in linked: + if link['expanded'] == element: + return link['compacted'] + + # do value compaction on @values and subject references + if _is_value(element) or _is_subject_reference(element): + rval = self._compact_value( + active_ctx, active_property, element) + if options['link'] and _is_subject_reference(element): + # store linked element + options['link'].setdefault(element['@id'], []).append( + {'expanded': element, 'compacted': rval}) + return rval + + # FIXME: avoid misuse of active property as an expanded property? + inside_reverse = (active_property == '@reverse') + + rval = {} + + if options['link'] and '@id' in element: + # store linked element + options['link'].setdefault(element['@id'], []).append( + {'expanded': element, 'compacted': rval}) + + # recursively process element keys in order + for expanded_property, expanded_value in sorted(element.items()): + # compact @id and @type(s) + if expanded_property == '@id' or expanded_property == '@type': + # compact single @id + if _is_string(expanded_value): + compacted_value = self._compact_iri( + active_ctx, expanded_value, + vocab=(expanded_property == '@type')) + # expanded value must be a @type array + else: + compacted_value = [] + for ev in expanded_value: + compacted_value.append(self._compact_iri( + active_ctx, ev, vocab=True)) + + # use keyword alias and add value + alias = self._compact_iri(active_ctx, expanded_property) + is_array = (_is_array(compacted_value) and + len(compacted_value) == 0) + JsonLdProcessor.add_value( + rval, alias, compacted_value, + {'propertyIsArray': is_array}) + continue + + # handle @reverse + if expanded_property == '@reverse': + # recursively compact expanded value + compacted_value = self._compact( + active_ctx, '@reverse', expanded_value, options) + + # handle double-reversed properties + for compacted_property, value in \ + list(compacted_value.items()): + mapping = active_ctx['mappings'].get( + compacted_property) + if mapping and mapping['reverse']: + container = JsonLdProcessor.get_context_value( + active_ctx, compacted_property, '@container') + use_array = (container == '@set' or + not options['compactArrays']) + JsonLdProcessor.add_value( + rval, compacted_property, value, + {'propertyIsArray': use_array}) + del compacted_value[compacted_property] + + if len(compacted_value.keys()) > 0: + # use keyword alias and add value + alias = self._compact_iri( + active_ctx, expanded_property) + JsonLdProcessor.add_value(rval, alias, compacted_value) + + continue + + # handle @index + if expanded_property == '@index': + # drop @index if inside an @index container + container = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@container') + if container == '@index': + continue + + # use keyword alias and add value + alias = self._compact_iri(active_ctx, expanded_property) + JsonLdProcessor.add_value(rval, alias, expanded_value) + continue + + # skip array processing for keywords that aren't + # @graph or @list + if(expanded_property != '@graph' and + expanded_property != '@list' and + _is_keyword(expanded_property)): + # use keyword alias and add value as is + alias = self._compact_iri(active_ctx, expanded_property) + JsonLdProcessor.add_value(rval, alias, expanded_value) + continue + + # Note: expanded value must be an array due to expansion + # algorithm. + + # preserve empty arrays + if len(expanded_value) == 0: + item_active_property = self._compact_iri( + active_ctx, expanded_property, expanded_value, + vocab=True, reverse=inside_reverse) + JsonLdProcessor.add_value( + rval, item_active_property, [], + {'propertyIsArray': True}) + + # recusively process array values + for expanded_item in expanded_value: + # compact property and get container type + item_active_property = self._compact_iri( + active_ctx, expanded_property, expanded_item, + vocab=True, reverse=inside_reverse) + container = JsonLdProcessor.get_context_value( + active_ctx, item_active_property, '@container') + + # get @list value if appropriate + is_list = _is_list(expanded_item) + list_ = None + if is_list: + list_ = expanded_item['@list'] + + # recursively compact expanded item + compacted_item = self._compact( + active_ctx, item_active_property, + list_ if is_list else expanded_item, options) + + # handle @list + if is_list: + # ensure @list is an array + compacted_item = JsonLdProcessor.arrayify( + compacted_item) + + if container != '@list': + # wrap using @list alias + wrapper = {} + wrapper[self._compact_iri( + active_ctx, '@list')] = compacted_item + compacted_item = wrapper + + # include @index from expanded @list, if any + if '@index' in expanded_item: + alias = self._compact_iri(active_ctx, '@index') + compacted_item[alias] = ( + expanded_item['@index']) + # can't use @list container for more than 1 list + elif item_active_property in rval: + raise JsonLdError( + 'JSON-LD compact error; property has a ' + '"@list" @container rule but there is more ' + 'than a single @list that matches the ' + 'compacted term in the document. Compaction ' + 'might mix unwanted items into the list.', + 'jsonld.SyntaxError', + code='compaction to list of lists') + + # handle language and index maps + if container == '@language' or container == '@index': + # get or create the map object + map_object = rval.setdefault(item_active_property, {}) + + # if container is a language map, simplify compacted + # value to a simple string + if (container == '@language' and + _is_value(compacted_item)): + compacted_item = compacted_item['@value'] + + # add compact value to map object using key from + # expanded value based on the container type + JsonLdProcessor.add_value( + map_object, expanded_item[container], + compacted_item) + else: + # use an array if compactArrays flag is false, + # @container is @set or @list, value is an empty + # array, or key is @graph + is_array = (not options['compactArrays'] or + container == '@set' or + container == '@list' or + (_is_array(compacted_item) and + len(compacted_item) == 0) or + expanded_property == '@list' or + expanded_property == '@graph') + + # add compact value + JsonLdProcessor.add_value( + rval, item_active_property, compacted_item, + {'propertyIsArray': is_array}) + + return rval + + # only primitives remain which are already compact + return element + + def _expand( + self, active_ctx, active_property, element, options, inside_list): + """ + Recursively expands an element using the given context. Any context in + the element will be removed. All context URLs must have been retrieved + before calling this method. + + :param active_ctx: the context to use. + :param active_property: the property for the element, None for none. + :param element: the element to expand. + :param options: the expansion options. + :param inside_list: True if the property is a list, False if not. + + :return: the expanded value. + """ + # nothing to expand + if element is None: + return element + + # recursively expand array + if _is_array(element): + rval = [] + container = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@container') + inside_list = inside_list or container == '@list' + for e in element: + # expand element + e = self._expand( + active_ctx, active_property, e, options, inside_list) + if inside_list and (_is_array(e) or _is_list(e)): + # lists of lists are illegal + raise JsonLdError( + 'Invalid JSON-LD syntax; lists of lists are not ' + 'permitted.', 'jsonld.SyntaxError', + code='list of lists') + # drop None values + if e is not None: + if _is_array(e): + rval.extend(e) + else: + rval.append(e) + return rval + + # handle scalars + if not _is_object(element): + # drop free-floating scalars that are not in lists + if (not inside_list and (active_property is None or + self._expand_iri( + active_ctx, active_property, vocab=True) == '@graph')): + return None + + # expand element according to value expansion rules + return self._expand_value(active_ctx, active_property, element) + + # recursively expand object + # if element has a context, process it + if '@context' in element: + active_ctx = self._process_context( + active_ctx, element['@context'], options) + + # expand the active property + expanded_active_property = self._expand_iri( + active_ctx, active_property, vocab=True) + + rval = {} + for key, value in sorted(element.items()): + if key == '@context': + continue + + # expand key to IRI + expanded_property = self._expand_iri( + active_ctx, key, vocab=True) + + # drop non-absolute IRI keys that aren't keywords + if (expanded_property is None or not + (_is_absolute_iri(expanded_property) or + _is_keyword(expanded_property))): + continue + + if _is_keyword(expanded_property): + if expanded_active_property == '@reverse': + raise JsonLdError( + 'Invalid JSON-LD syntax; a keyword cannot be used as ' + 'a @reverse property.', + 'jsonld.SyntaxError', {'value': value}, + code='invalid reverse property map') + if expanded_property in rval: + raise JsonLdError( + 'Invalid JSON-LD syntax; colliding keywords detected.', + 'jsonld.SyntaxError', {'keyword': expanded_property}, + code='colliding keywords') + + # syntax error if @id is not a string + if expanded_property == '@id' and not _is_string(value): + if not options.get('isFrame'): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@id" value must a string.', + 'jsonld.SyntaxError', {'value': value}, + code='invalid @id value') + if not _is_object(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@id" value must be a ' + 'string or an object.', 'jsonld.SyntaxError', + {'value': value}, code='invalid @id value') + + if expanded_property == '@type': + _validate_type_value(value) + + # @graph must be an array or an object + if (expanded_property == '@graph' and + not (_is_object(value) or _is_array(value))): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@graph" must not be an ' + 'object or an array.', 'jsonld.SyntaxError', + {'value': value}, code='invalid @graph value') + + # @value must not be an object or an array + if (expanded_property == '@value' and + (_is_object(value) or _is_array(value))): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@value" value must not be an ' + 'object or an array.', 'jsonld.SyntaxError', + {'value': value}, code='invalid value object value') + + # @language must be a string + if expanded_property == '@language': + if value is None: + # drop null @language values, they expand as if they + # didn't exist + continue + if not _is_string(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@language" value must be ' + 'a string.', 'jsonld.SyntaxError', {'value': value}, + code='invalid language-tagged string') + # ensure language value is lowercase + value = value.lower() + + # @index must be a string + if expanded_property == '@index' and not _is_string(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@index" value must be ' + 'a string.', 'jsonld.SyntaxError', {'value': value}, + code='invalid @index value') + + # reverse must be an object + if expanded_property == '@reverse': + if not _is_object(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@reverse" value must be ' + 'an object.', 'jsonld.SyntaxError', {'value': value}, + code='invalid @reverse value') + + expanded_value = self._expand( + active_ctx, '@reverse', value, options, inside_list) + + # properties double-reversed + if '@reverse' in expanded_value: + for rprop, rvalue in expanded_value['@reverse'].items(): + JsonLdProcessor.add_value( + rval, rprop, rvalue, + {'propertyIsArray': True}) + + # merge in all reversed properties + reverse_map = rval.get('@reverse') + for property, items in expanded_value.items(): + if property == '@reverse': + continue + if reverse_map is None: + reverse_map = rval['@reverse'] = {} + JsonLdProcessor.add_value( + reverse_map, property, [], + {'propertyIsArray': True}) + for item in items: + if _is_value(item) or _is_list(item): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@reverse" ' + 'value must not be an @value or an @list', + 'jsonld.SyntaxError', + {'value': expanded_value}, + code='invalid reverse property value') + JsonLdProcessor.add_value( + reverse_map, property, item, + {'propertyIsArray': True}) + + continue + + container = JsonLdProcessor.get_context_value( + active_ctx, key, '@container') + + # handle language map container (skip if value is not an object) + if container == '@language' and _is_object(value): + expanded_value = self._expand_language_map(value) + # handle index container (skip if value is not an object) + elif container == '@index' and _is_object(value): + def expand_index_map(active_property): + rval = [] + for k, v in sorted(value.items()): + v = self._expand( + active_ctx, active_property, + JsonLdProcessor.arrayify(v), + options, inside_list=False) + for item in v: + item.setdefault('@index', k) + rval.append(item) + return rval + expanded_value = expand_index_map(key) + else: + # recurse into @list or @set + is_list = (expanded_property == '@list') + if is_list or expanded_property == '@set': + next_active_property = active_property + if is_list and expanded_active_property == '@graph': + next_active_property = None + expanded_value = self._expand( + active_ctx, next_active_property, value, options, + is_list) + if is_list and _is_list(expanded_value): + raise JsonLdError( + 'Invalid JSON-LD syntax; lists of lists are ' + 'not permitted.', 'jsonld.SyntaxError', + code='list of lists') + else: + # recursively expand value w/key as new active property + expanded_value = self._expand( + active_ctx, key, value, options, inside_list=False) + + # drop None values if property is not @value (dropped below) + if expanded_value is None and expanded_property != '@value': + continue + + # convert expanded value to @list if container specifies it + if (expanded_property != '@list' and not _is_list(expanded_value) + and container == '@list'): + # ensure expanded value is an array + expanded_value = { + '@list': JsonLdProcessor.arrayify(expanded_value) + } + + # merge in reverse properties + mapping = active_ctx['mappings'].get(key) + if mapping and mapping['reverse']: + reverse_map = rval.setdefault('@reverse', {}) + expanded_value = JsonLdProcessor.arrayify(expanded_value) + for item in expanded_value: + if _is_value(item) or _is_list(item): + raise JsonLdError( + 'Invalid JSON-LD syntax; "@reverse" value must ' + 'not be an @value or an @list.', + 'jsonld.SyntaxError', {'value': expanded_value}, + code='invalid reverse property value') + JsonLdProcessor.add_value( + reverse_map, expanded_property, item, + {'propertyIsArray': True}) + continue + + # add value for property, use an array exception for certain + # key words + use_array = (expanded_property not in ['@index', '@id', '@type', + '@value', '@language']) + JsonLdProcessor.add_value( + rval, expanded_property, expanded_value, + {'propertyIsArray': use_array}) + + # get property count on expanded output + count = len(rval) + + if '@value' in rval: + # @value must only have @language or @type + if '@type' in rval and '@language' in rval: + raise JsonLdError( + 'Invalid JSON-LD syntax; an element containing ' + '"@value" may not contain both "@type" and "@language".', + 'jsonld.SyntaxError', {'element': rval}, + code='invalid value object') + valid_count = count - 1 + if '@type' in rval: + valid_count -= 1 + if '@index' in rval: + valid_count -= 1 + if '@language' in rval: + valid_count -= 1 + if valid_count != 0: + raise JsonLdError( + 'Invalid JSON-LD syntax; an element containing "@value" ' + 'may only have an "@index" property and at most one other ' + 'property which can be "@type" or "@language".', + 'jsonld.SyntaxError', {'element': rval}, + code='invalid value object') + # drop None @values + if rval['@value'] is None: + rval = None + # if @language is present, @value must be a string + elif '@language' in rval and not _is_string(rval['@value']): + raise JsonLdError( + 'Invalid JSON-LD syntax; only strings may be ' + 'language-tagged.', 'jsonld.SyntaxError', + {'element': rval}, code='invalid language-tagged value') + elif ('@type' in rval and (not _is_absolute_iri(rval['@type']) or + rval['@type'].startswith('_:'))): + raise JsonLdError( + 'Invalid JSON-LD syntax; an element containing "@value" ' + 'and "@type" must have an absolute IRI for the value ' + 'of "@type".', 'jsonld.SyntaxError', {'element': rval}, + code='invalid typed value') + # convert @type to an array + elif '@type' in rval and not _is_array(rval['@type']): + rval['@type'] = [rval['@type']] + # handle @set and @list + elif '@set' in rval or '@list' in rval: + if count > 1 and not (count == 2 and '@index' in rval): + raise JsonLdError( + 'Invalid JSON-LD syntax; if an element has the ' + 'property "@set" or "@list", then it can have at most ' + 'one other property, which is "@index".', + 'jsonld.SyntaxError', {'element': rval}, + code='invalid set or list object') + # optimize away @set + if '@set' in rval: + rval = rval['@set'] + count = len(rval) + # drop objects with only @language + elif count == 1 and '@language' in rval: + rval = None + + # drop certain top-level objects that do not occur in lists + if (_is_object(rval) and not options.get('keepFreeFloatingNodes') and + not inside_list and (active_property is None or + expanded_active_property == '@graph')): + # drop empty object or top-level @value/@list, + # or object with only @id + if (count == 0 or '@value' in rval or '@list' in rval or + (count == 1 and '@id' in rval)): + rval = None + + return rval + + def _flatten(self, input): + """ + Performs JSON-LD flattening. + + :param input_: the expanded JSON-LD to flatten. + + :return: the flattened JSON-LD output. + """ + # produce a map of all subjects and name each bnode + namer = UniqueNamer('_:b') + graphs = {'@default': {}} + self._create_node_map(input, graphs, '@default', namer) + + # add all non-default graphs to default graph + default_graph = graphs['@default'] + for graph_name, node_map in graphs.items(): + if graph_name == '@default': + continue + graph_subject = default_graph.setdefault( + graph_name, {'@id': graph_name, '@graph': []}) + graph_subject.setdefault('@graph', []).extend( + [v for k, v in sorted(node_map.items()) + if not _is_subject_reference(v)]) + + # produce flattened output + return [value for key, value in sorted(default_graph.items()) + if not _is_subject_reference(value)] + + def _frame(self, input_, frame, options): + """ + Performs JSON-LD framing. + + :param input_: the expanded JSON-LD to frame. + :param frame: the expanded JSON-LD frame to use. + :param options: the framing options. + + :return: the framed output. + """ + # create framing state + state = { + 'options': options, + 'graphs': {'@default': {}, '@merged': {}}, + 'subjectStack': [], + 'link': {} + } + + # produce a map of all graphs and name each bnode + # FIXME: currently uses subjects from @merged graph only + namer = UniqueNamer('_:b') + self._create_node_map(input_, state['graphs'], '@merged', namer) + state['subjects'] = state['graphs']['@merged'] + + # frame the subjects + framed = [] + self._match_frame( + state, sorted(state['subjects'].keys()), frame, framed, None) + return framed + + def _normalize(self, dataset, options): + """ + Performs RDF normalization on the given RDF dataset. + + :param dataset: the RDF dataset to normalize. + :param options: the normalization options. + + :return: the normalized output. + """ + # create quads and map bnodes to their associated quads + quads = [] + bnodes = {} + for graph_name, triples in dataset.items(): + if graph_name == '@default': + graph_name = None + for triple in triples: + quad = triple + if graph_name is not None: + if graph_name.startswith('_:'): + quad['name'] = {'type': 'blank node'} + else: + quad['name'] = {'type': 'IRI'} + quad['name']['value'] = graph_name + quads.append(quad) + + for attr in ['subject', 'object', 'name']: + if attr in quad and quad[attr]['type'] == 'blank node': + id_ = quad[attr]['value'] + bnodes.setdefault(id_, {}).setdefault( + 'quads', []).append(quad) + + # mapping complete, start canonical naming + namer = UniqueNamer('_:c14n') + + # continue to hash bnode quads while bnodes are assigned names + unnamed = None + next_unnamed = bnodes.keys() + duplicates = None + while True: + unnamed = next_unnamed + next_unnamed = [] + duplicates = {} + unique = {} + for bnode in unnamed: + # hash quads for each unnamed bnode + hash = self._hash_quads(bnode, bnodes) + + # store hash as unique or a duplicate + if hash in duplicates: + duplicates[hash].append(bnode) + next_unnamed.append(bnode) + elif hash in unique: + duplicates[hash] = [unique[hash], bnode] + next_unnamed.append(unique[hash]) + next_unnamed.append(bnode) + del unique[hash] + else: + unique[hash] = bnode + + # name unique bnodes in sorted hash order + for hash, bnode in sorted(unique.items()): + namer.get_name(bnode) + + # done when no more bnodes named + if len(unnamed) == len(next_unnamed): + break + + # enumerate duplicate hash groups in sorted order + for hash, group in sorted(duplicates.items()): + # process group + results = [] + for bnode in group: + # skip already-named bnodes + if namer.is_named(bnode): + continue + + # hash bnode paths + path_namer = UniqueNamer('_:b') + path_namer.get_name(bnode) + results.append(self._hash_paths( + bnode, bnodes, namer, path_namer)) + + # name bnodes in hash order + cmp_hashes = cmp_to_key(lambda x, y: cmp(x['hash'], y['hash'])) + for result in sorted(results, key=cmp_hashes): + # name all bnodes in path namer in key-entry order + for bnode in result['pathNamer'].order: + namer.get_name(bnode) + + # create normalized array + normalized = [] + + # Note: At this point all bnodes in the set of RDF quads have been + # assigned canonical names, which have been stored in the 'namer' + # object. Here each quad is updated by assigning each of its bnodes its + # new name via the 'namer' object. + + # update bnode names in each quad and serialize + for quad in quads: + for attr in ['subject', 'object', 'name']: + if (attr in quad and + quad[attr]['type'] == 'blank node' and + not quad[attr]['value'].startswith('_:c14n')): + quad[attr]['value'] = namer.get_name(quad[attr]['value']) + normalized.append(JsonLdProcessor.to_nquad( + quad, quad['name']['value'] if 'name' in quad else None)) + + # sort normalized output + normalized.sort() + + # handle output format + if 'format' in options: + if options['format'] == 'application/nquads': + return ''.join(normalized) + raise JsonLdError( + 'Unknown output format.', + 'jsonld.UnknownFormat', {'format': options['format']}) + + # return parsed RDF dataset + return JsonLdProcessor.parse_nquads(''.join(normalized)) + + def _from_rdf(self, dataset, options): + """ + Converts an RDF dataset to JSON-LD. + + :param dataset: the RDF dataset. + :param options: the RDF serialization options. + + :return: the JSON-LD output. + """ + default_graph = {} + graph_map = {'@default': default_graph} + referenced_once = {} + + for name, graph in dataset.items(): + graph_map.setdefault(name, {}) + if name != '@default' and name not in default_graph: + default_graph[name] = {'@id': name} + node_map = graph_map[name] + for triple in graph: + # get subject, predicate, object + s = triple['subject']['value'] + p = triple['predicate']['value'] + o = triple['object'] + + node = node_map.setdefault(s, {'@id': s}) + + object_is_id = (o['type'] == 'IRI' or + o['type'] == 'blank node') + if object_is_id and o['value'] not in node_map: + node_map[o['value']] = {'@id': o['value']} + + if (p == RDF_TYPE and not options.get('useRdfType', False) and + object_is_id): + JsonLdProcessor.add_value( + node, '@type', o['value'], {'propertyIsArray': True}) + continue + + value = self._rdf_to_object(o, options['useNativeTypes']) + JsonLdProcessor.add_value( + node, p, value, {'propertyIsArray': True}) + + # object may be an RDF list/partial list node but we + # can't know easily until all triples are read + if object_is_id: + # track rdf:nil uniquely per graph + if o['value'] == RDF_NIL: + object = node_map[o['value']] + if 'usages' not in object: + object['usages'] = [] + object['usages'].append({ + 'node': node, + 'property': p, + 'value': value + }) + # object referenced more than once + elif o['value'] in referenced_once: + referenced_once[o['value']] = False + # track single reference + else: + referenced_once[o['value']] = { + 'node': node, + 'property': p, + 'value': value + } + + # convert linked lists to @list arrays + for name, graph_object in graph_map.items(): + # no @lists to be converted, continue + if RDF_NIL not in graph_object: + continue + + # iterate backwards through each RDF list + nil = graph_object[RDF_NIL] + for usage in nil['usages']: + node = usage['node'] + property = usage['property'] + head = usage['value'] + list_ = [] + list_nodes = [] + + # ensure node is a well-formed list node; it must: + # 1. Be referenced only once. + # 2. Have an array for rdf:first that has 1 item. + # 3. Have an array for rdf:rest that has 1 item + # 4. Have no keys other than: @id, rdf:first, rdf:rest + # and, optionally, @type where the value is rdf:List. + node_key_count = len(node.keys()) + while(property == RDF_REST and + _is_object(referenced_once.get(node['@id'])) and + _is_array(node[RDF_FIRST]) and + len(node[RDF_FIRST]) == 1 and + _is_array(node[RDF_REST]) and + len(node[RDF_REST]) == 1 and + (node_key_count == 3 or (node_key_count == 4 and + _is_array(node.get('@type')) and + len(node['@type']) == 1 and + node['@type'][0] == RDF_LIST))): + list_.append(node[RDF_FIRST][0]) + list_nodes.append(node['@id']) + + # get next node, moving backwards through list + usage = referenced_once[node['@id']] + node = usage['node'] + property = usage['property'] + head = usage['value'] + node_key_count = len(node.keys()) + + # if node is not a blank node, then list head found + if not node['@id'].startswith('_:'): + break + + # the list is nested in another list + if property == RDF_FIRST: + # empty list + if node['@id'] == RDF_NIL: + # can't convert rdf:nil to a @list object because it + # would result in a list of lists which isn't supported + continue + + # preserve list head + head = graph_object[head['@id']][RDF_REST][0] + list_.pop() + list_nodes.pop() + + # transform list into @list object + del head['@id'] + list_.reverse() + head['@list'] = list_ + for node in list_nodes: + graph_object.pop(node, None) + + nil.pop('usages', None) + + result = [] + for subject, node in sorted(default_graph.items()): + if subject in graph_map: + graph = node['@graph'] = [] + for s, n in sorted(graph_map[subject].items()): + # only add full subjects to top-level + if not _is_subject_reference(n): + graph.append(n) + # only add full subjects to top-level + if not _is_subject_reference(node): + result.append(node) + + return result + + def _process_context(self, active_ctx, local_ctx, options): + """ + Processes a local context and returns a new active context. + + :param active_ctx: the current active context. + :param local_ctx: the local context to process. + :param options: the context processing options. + + :return: the new active context. + """ + global _cache + + # normalize local context to an array + if _is_object(local_ctx) and _is_array(local_ctx.get('@context')): + local_ctx = local_ctx['@context'] + ctxs = JsonLdProcessor.arrayify(local_ctx) + + # no contexts in array, clone existing context + if len(ctxs) == 0: + return self._clone_active_context(active_ctx) + + # process each context in order, update active context on each + # iteration to ensure proper caching + rval = active_ctx + for ctx in ctxs: + # reset to initial context + if ctx is None: + rval = active_ctx = self._get_initial_context(options) + continue + + # dereference @context key if present + if _is_object(ctx) and '@context' in ctx: + ctx = ctx['@context'] + + # context must be an object now, all URLs retrieved prior to call + if not _is_object(ctx): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context must be an object.', + 'jsonld.SyntaxError', {'context': ctx}, + code='invalid local context') + + # get context from cache if available + if _cache.get('activeCtx') is not None: + cached = _cache['activeCtx'].get(active_ctx, ctx) + if cached: + rval = active_ctx = cached + continue + + # update active context and clone new one before updating + active_ctx = rval + rval = self._clone_active_context(active_ctx) + + # define context mappings for keys in local context + defined = {} + + # handle @base + if '@base' in ctx: + base = ctx['@base'] + if base is None: + base = None + elif not _is_string(base): + raise JsonLdError( + 'Invalid JSON-LD syntax; the value of "@base" in a ' + '@context must be a string or null.', + 'jsonld.SyntaxError', {'context': ctx}, + code='invalid base IRI') + elif base != '' and not _is_absolute_iri(base): + raise JsonLdError( + 'Invalid JSON-LD syntax; the value of "@base" in a ' + '@context must be an absolute IRI or the empty ' + 'string.', 'jsonld.SyntaxError', {'context': ctx}, + code='invalid base IRI') + rval['@base'] = base + defined['@base'] = True + + # handle @vocab + if '@vocab' in ctx: + value = ctx['@vocab'] + if value is None: + del rval['@vocab'] + elif not _is_string(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; the value of "@vocab" in a ' + '@context must be a string or null.', + 'jsonld.SyntaxError', {'context': ctx}, + code='invalid vocab mapping') + elif not _is_absolute_iri(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; the value of "@vocab" in a ' + '@context must be an absolute IRI.', + 'jsonld.SyntaxError', {'context': ctx}, + code='invalid vocab mapping') + else: + rval['@vocab'] = value + defined['@vocab'] = True + + # handle @language + if '@language' in ctx: + value = ctx['@language'] + if value is None: + del rval['@language'] + elif not _is_string(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; the value of "@language" in ' + 'a @context must be a string or null.', + 'jsonld.SyntaxError', {'context': ctx}, + code='invalid default language') + else: + rval['@language'] = value.lower() + defined['@language'] = True + + # process all other keys + for k, v in ctx.items(): + self._create_term_definition(rval, ctx, k, defined) + + # cache result + if _cache.get('activeCtx') is not None: + _cache.get('activeCtx').set(active_ctx, ctx, rval) + + return rval + + def _expand_language_map(self, language_map): + """ + Expands a language map. + + :param language_map: the language map to expand. + + :return: the expanded language map. + """ + rval = [] + for key, values in sorted(language_map.items()): + values = JsonLdProcessor.arrayify(values) + for item in values: + if not _is_string(item): + raise JsonLdError( + 'Invalid JSON-LD syntax; language map values must be ' + 'strings.', 'jsonld.SyntaxError', + {'languageMap': language_map}, + code='invalid language map value') + rval.append({'@value': item, '@language': key.lower()}) + return rval + + def _expand_value(self, active_ctx, active_property, value): + """ + Expands the given value by using the coercion and keyword rules in the + given context. + + :param active_ctx: the active context to use. + :param active_property: the property the value is associated with. + :param value: the value to expand. + + :return: the expanded value. + """ + # nothing to expand + if value is None: + return None + + # special-case expand @id and @type (skips '@id' expansion) + expanded_property = self._expand_iri( + active_ctx, active_property, vocab=True) + if expanded_property == '@id': + return self._expand_iri(active_ctx, value, base=True) + elif expanded_property == '@type': + return self._expand_iri(active_ctx, value, vocab=True, base=True) + + # get type definition from context + type_ = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@type') + + # do @id expansion (automatic for @graph) + if (type_ == '@id' or (expanded_property == '@graph' + and _is_string(value))): + return {'@id': self._expand_iri(active_ctx, value, base=True)} + # do @id expansion w/vocab + if type_ == '@vocab': + return {'@id': self._expand_iri( + active_ctx, value, vocab=True, base=True)} + + # do not expand keyword values + if _is_keyword(expanded_property): + return value + + rval = {} + + # other type + if type_ is not None: + rval['@type'] = type_ + # check for language tagging + elif _is_string(value): + language = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@language') + if language is not None: + rval['@language'] = language + rval['@value'] = value + + return rval + + def _graph_to_rdf(self, graph, namer, options): + """ + Creates an array of RDF triples for the given graph. + + :param graph: the graph to create RDF triples for. + :param namer: the UniqueNamer for assigning blank node names. + :param options: the RDF serialization options. + + :return: the array of RDF triples for the given graph. + """ + rval = [] + for id_, node in sorted(graph.items()): + for property, items in sorted(node.items()): + if property == '@type': + property = RDF_TYPE + elif _is_keyword(property): + continue + + for item in items: + # skip relative IRI subjects and predicates + if not (_is_absolute_iri(id_) and + _is_absolute_iri(property)): + continue + + # RDF subject + subject = {} + if id_.startswith('_:'): + subject['type'] = 'blank node' + else: + subject['type'] = 'IRI' + subject['value'] = id_ + + # RDF predicate + predicate = {} + if property.startswith('_:'): + # skip bnode predicates unless producing + # generalized RDF + if not options['produceGeneralizedRdf']: + continue + predicate['type'] = 'blank node' + else: + predicate['type'] = 'IRI' + predicate['value'] = property + + # convert @list to triples + if _is_list(item): + self._list_to_rdf( + item['@list'], namer, subject, predicate, rval) + # convert value or node object to triple + else: + object = self._object_to_rdf(item) + # skip None objects (they are relative IRIs) + if object is not None: + rval.append({ + 'subject': subject, + 'predicate': predicate, + 'object': object + }) + return rval + + def _list_to_rdf(self, list, namer, subject, predicate, triples): + """ + Converts a @list value into a linked list of blank node RDF triples + (and RDF collection). + + :param list: the @list value. + :param namer: the UniqueNamer for assigning blank node names. + :param subject: the subject for the head of the list. + :param predicate: the predicate for the head of the list. + :param triples: the array of triples to append to. + """ + first = {'type': 'IRI', 'value': RDF_FIRST} + rest = {'type': 'IRI', 'value': RDF_REST} + nil = {'type': 'IRI', 'value': RDF_NIL} + + for item in list: + blank_node = {'type': 'blank node', 'value': namer.get_name()} + triples.append({ + 'subject': subject, + 'predicate': predicate, + 'object': blank_node + }) + + subject = blank_node + predicate = first + object = self._object_to_rdf(item) + # skip None objects (they are relative IRIs) + if object is not None: + triples.append({ + 'subject': subject, + 'predicate': predicate, + 'object': object + }) + + predicate = rest + + triples.append({ + 'subject': subject, + 'predicate': predicate, + 'object': nil + }) + + def _object_to_rdf(self, item): + """ + Converts a JSON-LD value object to an RDF literal or a JSON-LD string + or node object to an RDF resource. + + :param item: the JSON-LD value or node object. + + :return: the RDF literal or RDF resource. + """ + object = {} + + if _is_value(item): + object['type'] = 'literal' + value = item['@value'] + datatype = item.get('@type') + + # convert to XSD datatypes as appropriate + if _is_bool(value): + object['value'] = 'true' if value else 'false' + object['datatype'] = datatype or XSD_BOOLEAN + elif _is_double(value) or datatype == XSD_DOUBLE: + # canonical double representation + object['value'] = re.sub(r'(\d)0*E\+?0*(\d)', r'\1E\2', + ('%1.15E' % value)) + object['datatype'] = datatype or XSD_DOUBLE + elif _is_integer(value): + object['value'] = str(value) + object['datatype'] = datatype or XSD_INTEGER + elif '@language' in item: + object['value'] = value + object['datatype'] = datatype or RDF_LANGSTRING + object['language'] = item['@language'] + else: + object['value'] = value + object['datatype'] = datatype or XSD_STRING + # convert string/node object to RDF + else: + id_ = item['@id'] if _is_object(item) else item + if id_.startswith('_:'): + object['type'] = 'blank node' + else: + object['type'] = 'IRI' + object['value'] = id_ + + # skip relative IRIs + if object['type'] == 'IRI' and not _is_absolute_iri(object['value']): + return None + + return object + + def _rdf_to_object(self, o, use_native_types): + """ + Converts an RDF triple object to a JSON-LD object. + + :param o: the RDF triple object to convert. + :param use_native_types: True to output native types, False not to. + + :return: the JSON-LD object. + """ + # convert IRI/BlankNode object to JSON-LD + if o['type'] == 'IRI' or o['type'] == 'blank node': + return {'@id': o['value']} + + # convert literal object to JSON-LD + rval = {'@value': o['value']} + + # add language + if 'language' in o: + rval['@language'] = o['language'] + # add datatype + else: + type_ = o['datatype'] + # use native types for certain xsd types + if use_native_types: + if type_ == XSD_BOOLEAN: + if rval['@value'] == 'true': + rval['@value'] = True + elif rval['@value'] == 'false': + rval['@value'] = False + elif _is_numeric(rval['@value']): + if type_ == XSD_INTEGER: + if rval['@value'].isdigit(): + rval['@value'] = int(rval['@value']) + elif type_ == XSD_DOUBLE: + rval['@value'] = float(rval['@value']) + # do not add native type + if type_ not in [XSD_BOOLEAN, XSD_INTEGER, XSD_DOUBLE, + XSD_STRING]: + rval['@type'] = type_ + elif type_ != XSD_STRING: + rval['@type'] = type_ + return rval + + def _create_node_map( + self, input_, graphs, graph, namer, name=None, list_=None): + """ + Recursively flattens the subjects in the given JSON-LD expanded + input into a node map. + + :param input_: the JSON-LD expanded input. + :param graphs: a map of graph name to subject map. + :param graph: the name of the current graph. + :param namer: the UniqueNamer for assigning blank node names. + :param name: the name assigned to the current input if it is a bnode. + :param list_: the list to append to, None for none. + """ + # recurse through array + if _is_array(input_): + for e in input_: + self._create_node_map(e, graphs, graph, namer, None, list_) + return + + # add non-object to list + if not _is_object(input_): + if list_ is not None: + list_.append(input_) + return + + # add values to list + if _is_value(input_): + if '@type' in input_: + type_ = input_['@type'] + # rename @type blank node + if type_.startswith('_:'): + type_ = input_['@type'] = namer.get_name(type_) + if list_ is not None: + list_.append(input_) + return + + # Note: At this point, input must be a subject. + + # spec requires @type to be named first, so assign names early + if '@type' in input_: + for type_ in input_['@type']: + if type_.startswith('_:'): + namer.get_name(type_) + + # get name for subject + if name is None: + name = input_.get('@id') + if _is_bnode(input_): + name = namer.get_name(name) + + # add subject reference to list + if list_ is not None: + list_.append({'@id': name}) + + # create new subject or merge into existing one + subject = graphs.setdefault(graph, {}).setdefault(name, {'@id': name}) + for property, objects in sorted(input_.items()): + # skip @id + if property == '@id': + continue + + # handle reverse properties + if property == '@reverse': + referenced_node = {'@id': name} + reverse_map = input_['@reverse'] + for reverse_property, items in reverse_map.items(): + for item in items: + item_name = item.get('@id') + if _is_bnode(item): + item_name = namer.get_name(item_name) + self._create_node_map( + item, graphs, graph, namer, item_name) + JsonLdProcessor.add_value( + graphs[graph][item_name], reverse_property, + referenced_node, + {'propertyIsArray': True, 'allowDuplicate': False}) + continue + + # recurse into graph + if property == '@graph': + # add graph subjects map entry + graphs.setdefault(name, {}) + g = graph if graph == '@merged' else name + self._create_node_map(objects, graphs, g, namer) + continue + + # copy non-@type keywords + if property != '@type' and _is_keyword(property): + if property == '@index' and '@index' in subject \ + and (input_['@index'] != subject['@index'] or + input_['@index']['@id'] != subject['@index']['@id']): + raise JsonLdError( + 'Invalid JSON-LD syntax; conflicting @index property ' + ' detected.', 'jsonld.SyntaxError', + {'subject': subject}, code='conflicting indexes') + subject[property] = input_[property] + continue + + # if property is a bnode, assign it a new id + if property.startswith('_:'): + property = namer.get_name(property) + + # ensure property is added for empty arrays + if len(objects) == 0: + JsonLdProcessor.add_value( + subject, property, [], {'propertyIsArray': True}) + continue + + for o in objects: + if property == '@type': + # rename @type blank nodes + o = namer.get_name(o) if o.startswith('_:') else o + + # handle embedded subject or subject reference + if _is_subject(o) or _is_subject_reference(o): + # rename blank node @id + id_ = o.get('@id') + if _is_bnode(o): + id_ = namer.get_name(id_) + + # add reference and recurse + JsonLdProcessor.add_value( + subject, property, {'@id': id_}, + {'propertyIsArray': True, 'allowDuplicate': False}) + self._create_node_map(o, graphs, graph, namer, id_) + # handle @list + elif _is_list(o): + olist = [] + self._create_node_map( + o['@list'], graphs, graph, namer, name, olist) + o = {'@list': olist} + JsonLdProcessor.add_value( + subject, property, o, + {'propertyIsArray': True, 'allowDuplicate': False}) + # handle @value + else: + self._create_node_map(o, graphs, graph, namer, name) + JsonLdProcessor.add_value( + subject, property, o, + {'propertyIsArray': True, 'allowDuplicate': False}) + + def _match_frame(self, state, subjects, frame, parent, property): + """ + Frames subjects according to the given frame. + + :param state: the current framing state. + :param subjects: the subjects to filter. + :param frame: the frame. + :param parent: the parent subject or top-level array. + :param property: the parent property, initialized to None. + """ + # validate the frame + self._validate_frame(frame) + frame = frame[0] + + # get flags for current frame + options = state['options'] + flags = { + 'embed': self._get_frame_flag(frame, options, 'embed'), + 'explicit': self._get_frame_flag(frame, options, 'explicit'), + 'requireAll': self._get_frame_flag(frame, options, 'requireAll') + } + + # filter out subjects that match the frame + matches = self._filter_subjects(state, subjects, frame, flags) + + # add matches to output + for id_, subject in sorted(matches.items()): + if flags['embed'] == '@link' and id_ in state['link']: + # TODO: may want to also match an existing linked subject + # against the current frame ... so different frames could + # produce different subjects that are only shared in-memory + # when the frames are the same + + # add existing linked subject + self._add_frame_output(parent, property, state['link'][id_]) + continue + + # Note: In order to treat each top-level match as a + # compartmentalized result, clear the unique embedded subjects map + # when the property is None, which only occurs at the top-level. + if property is None: + state['uniqueEmbeds'] = {} + + # start output for subject + output = {'@id': id_} + state['link'][id_] = output + + # if embed is @never or if a circular reference would be created + # by an embed, the subject cannot be embedded, just add the + # reference; note that a circular reference won't occur when the + # embed flag is `@link` as the above check will short-circuit + # before reaching this point + if(flags['embed'] == '@never' or self._creates_circular_reference( + subject, state['subjectStack'])): + self._add_frame_output(parent, property, output) + continue + + # if only the last match should be embedded + if flags['embed'] == '@last': + # remove any existing embed + if id_ in state['uniqueEmbeds']: + self._remove_embed(state, id_) + state['uniqueEmbeds'][id_] = { + 'parent': parent, + 'property': property + } + + # push matching subject onto stack to enable circular embed checks + state['subjectStack'].append(subject) + + # iterate over subject properties in order + for prop, objects in sorted(subject.items()): + # copy keywords to output + if _is_keyword(prop): + output[prop] = copy.deepcopy(subject[prop]) + continue + + # explicit is on and property isn't in frame, skip processing + if flags['explicit'] and prop not in frame: + continue + + # add objects + objects = subject[prop] + for o in objects: + # recurse into list + if _is_list(o): + # add empty list + list_ = {'@list': []} + self._add_frame_output(output, prop, list_) + + # add list objects + src = o['@list'] + for o in src: + if _is_subject_reference(o): + # recurse into subject reference + if prop in frame: + subframe = frame[prop][0]['@list'] + else: + subframe = self._create_implicit_frame( + flags) + self._match_frame( + state, [o['@id']], + subframe, list_, '@list') + else: + # include other values automatically + self._add_frame_output( + list_, '@list', copy.deepcopy(o)) + continue + + if _is_subject_reference(o): + # recurse into subject reference + if prop in frame: + subframe = frame[prop] + else: + subframe = self._create_implicit_frame(flags) + self._match_frame( + state, [o['@id']], subframe, output, prop) + else: + # include other values automatically + self._add_frame_output(output, prop, copy.deepcopy(o)) + + # handle defaults in order + for prop in sorted(frame.keys()): + # skip keywords + if _is_keyword(prop): + continue + # if omit default is off, then include default values for + # properties that appear in the next frame but are not in + # the matching subject + next = frame[prop][0] + omit_default_on = self._get_frame_flag( + next, options, 'omitDefault') + if not omit_default_on and prop not in output: + preserve = '@null' + if '@default' in next: + preserve = copy.deepcopy(next['@default']) + preserve = JsonLdProcessor.arrayify(preserve) + output[prop] = [{'@preserve': preserve}] + + # add output to parent + self._add_frame_output(parent, property, output) + + # pop matching subject from circular ref-checking stack + state['subjectStack'].pop() + + def _create_implicit_frame(self, flags): + """ + Creates an implicit frame when recursing through subject matches. If + a frame doesn't have an explicit frame for a particular property, then + a wildcard child frame will be created that uses the same flags that + the parent frame used. + + :param flags: the current framing flags. + + :return: the implicit frame. + """ + frame = {} + for key in flags: + frame['@' + key] = [flags[key]] + return [frame] + + def _creates_circular_reference(self, subject_to_embed, subject_stack): + """ + Checks the current subject stack to see if embedding the given subject + would cause a circular reference. + + :param subject_to_embed: the subject to embed. + :param subject_stack: the current stack of subjects. + + :return: true if a circular reference would be created, false if not. + """ + for subject in reversed(subject_stack[:-1]): + if subject['@id'] == subject_to_embed['@id']: + return True + return False + + def _get_frame_flag(self, frame, options, name): + """ + Gets the frame flag value for the given flag name. + + :param frame: the frame. + :param options: the framing options. + :param name: the flag name. + + :return: the flag value. + """ + rval = frame.get('@' + name, [options[name]])[0] + if name == 'embed': + # default is "@last" + # backwards-compatibility support for "embed" maps: + # true => "@last" + # false => "@never" + if rval is True: + rval = '@last' + elif rval is False: + rval = '@never' + elif rval != '@always' and rval != '@never' and rval != '@link': + rval = '@last' + return rval + + def _validate_frame(self, frame): + """ + Validates a JSON-LD frame, throwing an exception if the frame is + invalid. + + :param frame: the frame to validate. + """ + if (not _is_array(frame) or len(frame) != 1 or + not _is_object(frame[0])): + raise JsonLdError( + 'Invalid JSON-LD syntax; a JSON-LD frame must be a single ' + 'object.', 'jsonld.SyntaxError', {'frame': frame}) + + def _filter_subjects(self, state, subjects, frame, flags): + """ + Returns a map of all of the subjects that match a parsed frame. + + :param state: the current framing state. + :param subjects: the set of subjects to filter. + :param frame: the parsed frame. + :param flags: the frame flags. + + :return: all of the matched subjects. + """ + rval = {} + for id_ in subjects: + subject = state['subjects'][id_] + if self._filter_subject(subject, frame, flags): + rval[id_] = subject + return rval + + def _filter_subject(self, subject, frame, flags): + """ + Returns True if the given subject matches the given frame. + + :param subject: the subject to check. + :param frame: the frame to check. + :param flags: the frame flags. + + :return: True if the subject matches, False if not. + """ + # check @type (object value means 'any' type, fall through to + # ducktyping) + if ('@type' in frame and + not (len(frame['@type']) == 1 and + _is_object(frame['@type'][0]))): + types = frame['@type'] + for t in types: + # any matching @type is a match + if JsonLdProcessor.has_value(subject, '@type', t): + return True + return False + + # check ducktype + wildcard = True + matches_some = False + for k, v in frame.items(): + if _is_keyword(k): + # skip non-@id and non-@type + if k != '@id' and k != '@type': + continue + wildcard = True + + # check @id for a specific @id value + if k == '@id' and _is_string(v): + if subject.get(k) != v: + return False + + matches_some = True + continue + + wildcard = False + + if k in subject: + # v == [] means do not match if property is present + if _is_array(v) and len(v) == 0: + return False + + matches_some = True + continue + + # all properties must match to be a duck unless a @default is + # specified + has_default = (_is_array(v) and len(v) == 1 and + _is_object(v[0]) and '@default' in v[0]) + if flags['requireAll'] and not has_default: + return False + + # return true if wildcard or subject matches some properties + return wildcard or matches_some + + def _remove_embed(self, state, id_): + """ + Removes an existing embed. + + :param state: the current framing state. + :param id_: the @id of the embed to remove. + """ + # get existing embed + embeds = state['uniqueEmbeds'] + embed = embeds[id_] + property = embed['property'] + + # create reference to replace embed + subject = {'@id': id_} + + # remove existing embed + if _is_array(embed['parent']): + # replace subject with reference + for i, parent in enumerate(embed['parent']): + if JsonLdProcessor.compare_values(parent, subject): + embed['parent'][i] = subject + break + else: + # replace subject with reference + use_array = _is_array(embed['parent'][property]) + JsonLdProcessor.remove_value( + embed['parent'], property, subject, + {'propertyIsArray': use_array}) + JsonLdProcessor.add_value( + embed['parent'], property, subject, + {'propertyIsArray': use_array}) + + # recursively remove dependent dangling embeds + def remove_dependents(id_): + # get embed keys as a separate array to enable deleting keys + # in map + try: + ids = list(embeds.iterkeys()) + except AttributeError: + ids = list(embeds.keys()) + for next in ids: + if (next in embeds and + _is_object(embeds[next]['parent']) and + embeds[next]['parent']['@id'] == id_): + del embeds[next] + remove_dependents(next) + remove_dependents(id_) + + def _add_frame_output(self, parent, property, output): + """ + Adds framing output to the given parent. + + :param parent: the parent to add to. + :param property: the parent property. + :param output: the output to add. + """ + if _is_object(parent): + JsonLdProcessor.add_value( + parent, property, output, {'propertyIsArray': True}) + else: + parent.append(output) + + def _remove_preserve(self, ctx, input_, options): + """ + Removes the @preserve keywords as the last step of the framing + algorithm. + + :param ctx: the active context used to compact the input. + :param input_: the framed, compacted output. + :param options: the compaction options used. + + :return: the resulting output. + """ + # recurse through arrays + if _is_array(input_): + output = [] + for e in input_: + result = self._remove_preserve(ctx, e, options) + # drop Nones from arrays + if result is not None: + output.append(result) + return output + elif _is_object(input_): + # remove @preserve + if '@preserve' in input_: + if input_['@preserve'] == '@null': + return None + return input_['@preserve'] + + # skip @values + if _is_value(input_): + return input_ + + # recurse through @lists + if _is_list(input_): + input_['@list'] = self._remove_preserve( + ctx, input_['@list'], options) + return input_ + + # handle in-memory linked nodes + id_alias = self._compact_iri(ctx, '@id') + if id_alias in input_: + id_ = input_[id_alias] + if id_ in options['link']: + try: + idx = options['link'][id_].index(input_) + # already visited + return options['link'][id_][idx] + except BaseException: + # prevent circular visitation + options['link'][id_].append(input_) + else: + # prevent circular visitation + options['link'][id_] = [input_] + + # recurse through properties + for prop, v in input_.items(): + result = self._remove_preserve(ctx, v, options) + container = JsonLdProcessor.get_context_value( + ctx, prop, '@container') + if (options['compactArrays'] and + _is_array(result) and len(result) == 1 and + container != '@set' and container != '@list'): + result = result[0] + input_[prop] = result + return input_ + + def _hash_quads(self, id_, bnodes): + """ + Hashes all of the quads about a blank node. + + :param id_: the ID of the bnode to hash quads for. + :param bnodes: the mapping of bnodes to quads. + :param namer: the canonical bnode namer. + + :return: the new hash. + """ + # return cached hash + if 'hash' in bnodes[id_]: + return bnodes[id_]['hash'] + + # serialize all of bnode's quads + quads = bnodes[id_]['quads'] + nquads = [] + for quad in quads: + nquads.append(JsonLdProcessor.to_nquad( + quad, quad['name']['value'] if 'name' in quad else None, id_)) + # sort serialized quads + nquads.sort() + # cache and return hashed quads + md = hashlib.sha1() + md.update(''.join(nquads).encode('utf-8')) + hash = bnodes[id_]['hash'] = md.hexdigest() + return hash + + def _hash_paths(self, id_, bnodes, namer, path_namer): + """ + Produces a hash for the paths of adjacent bnodes for a bnode, + incorporating all information about its subgraph of bnodes. This + method will recursively pick adjacent bnode permutations that produce + the lexicographically-least 'path' serializations. + + :param id_: the ID of the bnode to hash paths for. + :param bnodes: the map of bnode quads. + :param namer: the canonical bnode namer. + :param path_namer: the namer used to assign names to adjacent bnodes. + + :return: the hash and path namer used. + """ + # create SHA-1 digest + md = hashlib.sha1() + + # group adjacent bnodes by hash, keep properties & references separate + groups = {} + quads = bnodes[id_]['quads'] + for quad in quads: + # get adjacent bnode + bnode = self._get_adjacent_bnode_name(quad['subject'], id_) + if bnode is not None: + # normal property + direction = 'p' + else: + bnode = self._get_adjacent_bnode_name(quad['object'], id_) + if bnode is None: + continue + # reference property + direction = 'r' + + # get bnode name (try canonical, path, then hash) + if namer.is_named(bnode): + name = namer.get_name(bnode) + elif path_namer.is_named(bnode): + name = path_namer.get_name(bnode) + else: + name = self._hash_quads(bnode, bnodes) + + # hash direction, property, and bnode name/hash + group_md = hashlib.sha1() + group_md.update(direction.encode('utf-8')) + group_md.update(quad['predicate']['value'].encode('utf-8')) + group_md.update(name.encode('utf-8')) + group_hash = group_md.hexdigest() + + # add bnode to hash group + groups.setdefault(group_hash, []).append(bnode) + + # iterate over groups in sorted hash order + for group_hash, group in sorted(groups.items()): + # digest group hash + md.update(group_hash.encode('utf8')) + + # choose a path and namer from the permutations + chosen_path = None + chosen_namer = None + for permutation in permutations(group): + path_namer_copy = copy.deepcopy(path_namer) + + # build adjacent path + path = '' + skipped = False + recurse = [] + for bnode in permutation: + # use canonical name if available + if namer.is_named(bnode): + path += namer.get_name(bnode) + else: + # recurse if bnode isn't named in the path yet + if not path_namer_copy.is_named(bnode): + recurse.append(bnode) + path += path_namer_copy.get_name(bnode) + + # skip permutation if path is already >= chosen path + if (chosen_path is not None and + len(path) >= len(chosen_path) and + path > chosen_path): + skipped = True + break + + # recurse + if not skipped: + for bnode in recurse: + result = self._hash_paths( + bnode, bnodes, namer, path_namer_copy) + path += path_namer_copy.get_name(bnode) + path += '<%s>' % result['hash'] + path_namer_copy = result['pathNamer'] + + # skip permutation if path is already >= chosen path + if (chosen_path is not None and + len(path) >= len(chosen_path) and + path > chosen_path): + skipped = True + break + + if (not skipped and + (chosen_path is None or path < chosen_path)): + chosen_path = path + chosen_namer = path_namer_copy + + # digest chosen path and update namer + md.update(chosen_path.encode('utf-8')) + path_namer = chosen_namer + + # return SHA-1 hash and path namer + return {'hash': md.hexdigest(), 'pathNamer': path_namer} + + def _get_adjacent_bnode_name(self, node, id_): + """ + A helper function that gets the blank node name from an RDF quad + node (subject or object). If the node is not a blank node or its + value does not match the given blank node ID, it will be returned. + + :param node: the RDF quad node. + :param id_: the ID of the blank node to look next to. + + :return: the adjacent blank node name or None if none was found. + """ + if node['type'] == 'blank node' and node['value'] != id_: + return node['value'] + return None + + def _select_term( + self, active_ctx, iri, value, containers, + type_or_language, type_or_language_value): + """ + Picks the preferred compaction term from the inverse context entry. + + :param active_ctx: the active context. + :param iri: the IRI to pick the term for. + :param value: the value to pick the term for. + :param containers: the preferred containers. + :param type_or_language: either '@type' or '@language'. + :param type_or_language_value: the preferred value for '@type' or + '@language' + + :return: the preferred term. + """ + if type_or_language_value is None: + type_or_language_value = '@null' + + # preferred options for the value of @type or language + prefs = [] + + # determine prefs for @id based on whether value compacts to term + if ((type_or_language_value == '@id' or + type_or_language_value == '@reverse') and + _is_subject_reference(value)): + # prefer @reverse first + if type_or_language_value == '@reverse': + prefs.append('@reverse') + # try to compact value to a term + term = self._compact_iri( + active_ctx, value['@id'], None, vocab=True) + mapping = active_ctx['mappings'].get(term) + if term is not None and mapping and mapping['@id'] == value['@id']: + # prefer @vocab + prefs.extend(['@vocab', '@id']) + else: + # prefer @id + prefs.extend(['@id', '@vocab']) + else: + prefs.append(type_or_language_value) + prefs.append('@none') + + container_map = active_ctx['inverse'][iri] + for container in containers: + # skip container if not in map + if container not in container_map: + continue + type_or_language_value_map = ( + container_map[container][type_or_language]) + for pref in prefs: + # skip type/language preference if not in map + if pref not in type_or_language_value_map: + continue + return type_or_language_value_map[pref] + return None + + def _compact_iri( + self, active_ctx, iri, value=None, vocab=False, reverse=False): + """ + Compacts an IRI or keyword into a term or CURIE if it can be. If the + IRI has an associated value it may be passed. + + :param active_ctx: the active context to use. + :param iri: the IRI to compact. + :param value: the value to check or None. + :param vocab: True to compact using @vocab if available, False not to. + :param reverse: True if a reverse property is being compacted, False if + not. + + :return: the compacted term, prefix, keyword alias, or original IRI. + """ + # can't compact None + if iri is None: + return iri + + # term is a keyword, force vocab to True + if _is_keyword(iri): + vocab = True + + # use inverse context to pick a term if iri is relative to vocab + if vocab and iri in self._get_inverse_context(active_ctx): + default_language = active_ctx.get('@language', '@none') + + # prefer @index if available in value + containers = [] + if _is_object(value) and '@index' in value: + containers.append('@index') + + # defaults for term selection based on type/language + type_or_language = '@language' + type_or_language_value = '@null' + + if reverse: + type_or_language = '@type' + type_or_language_value = '@reverse' + containers.append('@set') + # choose most specific term that works for all elements in @list + elif _is_list(value): + # only select @list containers if @index is NOT in value + if '@index' not in value: + containers.append('@list') + list_ = value['@list'] + common_language = default_language if len(list_) == 0 else None + common_type = None + for item in list_: + item_language = '@none' + item_type = '@none' + if _is_value(item): + if '@language' in item: + item_language = item['@language'] + elif '@type' in item: + item_type = item['@type'] + # plain literal + else: + item_language = '@null' + else: + item_type = '@id' + if common_language is None: + common_language = item_language + elif item_language != common_language and _is_value(item): + common_language = '@none' + if common_type is None: + common_type = item_type + elif item_type != common_type: + common_type = '@none' + # there are different languages and types in the list, so + # choose the most generic term, no need to keep iterating + if common_language == '@none' and common_type == '@none': + break + if common_language is None: + common_language = '@none' + if common_type is None: + common_type = '@none' + if common_type != '@none': + type_or_language = '@type' + type_or_language_value = common_type + else: + type_or_language_value = common_language + # non-@list + else: + if _is_value(value): + if '@language' in value and '@index' not in value: + containers.append('@language') + type_or_language_value = value['@language'] + elif '@type' in value: + type_or_language = '@type' + type_or_language_value = value['@type'] + else: + type_or_language = '@type' + type_or_language_value = '@id' + containers.append('@set') + + # do term selection + containers.append('@none') + term = self._select_term( + active_ctx, iri, value, containers, + type_or_language, type_or_language_value) + if term is not None: + return term + + # no term match, use @vocab if available + if vocab: + if '@vocab' in active_ctx: + vocab_ = active_ctx['@vocab'] + if iri.startswith(vocab_) and iri != vocab_: + # use suffix as relative iri if it is not a term in the + # active context + suffix = iri[len(vocab_):] + if suffix not in active_ctx['mappings']: + return suffix + + # no term or @vocab match, check for possible CURIEs + candidate = None + for term, definition in active_ctx['mappings'].items(): + # skip terms with colons, they can't be prefixes + if ':' in term: + continue + # skip entries with @ids that are not partial matches + if (definition is None or definition['@id'] == iri or + not iri.startswith(definition['@id'])): + continue + + # a CURIE is usable if: + # 1. it has no mapping, OR + # 2. value is None, which means we're not compacting an @value, AND + # the mapping matches the IRI + curie = term + ':' + iri[len(definition['@id']):] + is_usable_curie = ( + curie not in active_ctx['mappings'] or + (value is None and + active_ctx['mappings'].get(curie, {}).get('@id') == iri)) + + # select curie if it is shorter or the same length but + # lexicographically less than the current choice + if (is_usable_curie and (candidate is None or + _compare_shortest_least(curie, + candidate) < 0)): + candidate = curie + + # return curie candidate + if candidate is not None: + return candidate + + # compact IRI relative to base + if not vocab: + return remove_base(active_ctx['@base'], iri) + + # return IRI as is + return iri + + def _compact_value(self, active_ctx, active_property, value): + """ + Performs value compaction on an object with @value or @id as the only + property. + + :param active_ctx: the active context. + :param active_property: the active property that points to the value. + :param value: the value to compact. + """ + if _is_value(value): + # get context rules + type_ = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@type') + language = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@language') + container = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@container') + + # whether or not the value has an @index that must be preserved + preserve_index = '@index' in value and container != '@index' + + # if there's no @index to preserve + if not preserve_index: + # matching @type or @language specified in context, compact + if (('@type' in value and value['@type'] == type_) or + ('@language' in value and + value['@language'] == language)): + return value['@value'] + + # return just the value of @value if all are true: + # 1. @value is the only key or @index isn't being preserved + # 2. there is no default language or @value is not a string or + # the key has a mapping with a null @language + key_count = len(value) + is_value_only_key = \ + (key_count == 1 or (key_count == 2 and + '@index' in value and not preserve_index)) + has_default_language = '@language' in active_ctx + is_value_string = _is_string(value['@value']) + has_null_mapping = ( + active_ctx['mappings'].get(active_property) is not None and + '@language' in active_ctx['mappings'][active_property] and + active_ctx['mappings'][active_property]['@language'] is None) + if (is_value_only_key and ( + not has_default_language or not is_value_string or + has_null_mapping)): + return value['@value'] + + rval = {} + + # preserve @index + if preserve_index: + rval[self._compact_iri(active_ctx, '@index')] = value['@index'] + + # compact @type IRI + if '@type' in value: + rval[self._compact_iri(active_ctx, '@type')] = ( + self._compact_iri(active_ctx, value['@type'], vocab=True)) + # alias @language + elif '@language' in value: + rval[self._compact_iri(active_ctx, '@language')] = ( + value['@language']) + + # alias @value + rval[self._compact_iri(active_ctx, '@value')] = value['@value'] + + return rval + + # value is a subject reference + expanded_property = self._expand_iri( + active_ctx, active_property, vocab=True) + type_ = JsonLdProcessor.get_context_value( + active_ctx, active_property, '@type') + compacted = self._compact_iri( + active_ctx, value['@id'], vocab=(type_ == '@vocab')) + + # compact to scalar + if type_ in ['@id', '@vocab'] or expanded_property == '@graph': + return compacted + + rval = {} + rval[self._compact_iri(active_ctx, '@id')] = compacted + return rval + + def _create_term_definition(self, active_ctx, local_ctx, term, defined): + """ + Creates a term definition during context processing. + + :param active_ctx: the current active context. + :param local_ctx: the local context being processed. + :param term: the key in the local context to define the mapping for. + :param defined: a map of defining/defined keys to detect cycles + and prevent double definitions. + """ + if term in defined: + # term already defined + if defined[term]: + return + # cycle detected + raise JsonLdError( + 'Cyclical context definition detected.', + 'jsonld.CyclicalContext', { + 'context': local_ctx, + 'term': term + }, code='cyclic IRI mapping') + + # now defining term + defined[term] = False + + if _is_keyword(term): + raise JsonLdError( + 'Invalid JSON-LD syntax; keywords cannot be overridden.', + 'jsonld.SyntaxError', {'context': local_ctx, 'term': term}, + code='keyword redefinition') + + if term == '': + raise JsonLdError( + 'Invalid JSON-LD syntax; a term cannot be an empty string.', + 'jsonld.SyntaxError', {'context': local_ctx}, + code='invalid term definition') + + # remove old mapping + if term in active_ctx['mappings']: + del active_ctx['mappings'][term] + + # get context term value + value = local_ctx[term] + + # clear context entry + if (value is None or (_is_object(value) and '@id' in value and + value['@id'] is None)): + active_ctx['mappings'][term] = None + defined[term] = True + return + + # convert short-hand value to object w/@id + if _is_string(value): + value = {'@id': value} + + if not _is_object(value): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context property values must be ' + 'strings or objects.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid term definition') + + # create new mapping + mapping = active_ctx['mappings'][term] = {'reverse': False} + + if '@reverse' in value: + if '@id' in value: + raise JsonLdError( + 'Invalid JSON-LD syntax; an @reverse term definition must ' + 'not contain @id.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid reverse property') + reverse = value['@reverse'] + if not _is_string(reverse): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @reverse value must be ' + 'a string.', 'jsonld.SyntaxError', {'context': local_ctx}, + code='invalid IRI mapping') + + # expand and add @id mapping + id_ = self._expand_iri( + active_ctx, reverse, vocab=True, base=False, + local_ctx=local_ctx, defined=defined) + if not _is_absolute_iri(id_): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @reverse value must be ' + 'an absolute IRI or a blank node identifier.', + 'jsonld.SyntaxError', {'context': local_ctx}, + code='invalid IRI mapping') + mapping['@id'] = id_ + mapping['reverse'] = True + elif '@id' in value: + id_ = value['@id'] + if not _is_string(id_): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @id value must be a ' + 'string.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid IRI mapping') + if id_ != term: + # add @id to mapping + id_ = self._expand_iri( + active_ctx, id_, vocab=True, base=False, + local_ctx=local_ctx, defined=defined) + if not _is_absolute_iri(id_) and not _is_keyword(id_): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @id value must be ' + 'an absolute IRI, a blank node identifier, or a ' + 'keyword.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid IRI mapping') + mapping['@id'] = id_ + if '@id' not in mapping: + # see if the term has a prefix + colon = term.find(':') + if colon != -1: + prefix = term[0:colon] + if prefix in local_ctx: + # define parent prefix + self._create_term_definition( + active_ctx, local_ctx, prefix, defined) + + # set @id based on prefix parent + if active_ctx['mappings'].get(prefix) is not None: + suffix = term[colon + 1:] + mapping['@id'] = (active_ctx['mappings'][prefix]['@id'] + + suffix) + # term is an absolute IRI + else: + mapping['@id'] = term + else: + # non-IRIs MUST define @ids if @vocab not available + if '@vocab' not in active_ctx: + raise JsonLdError( + 'Invalid JSON-LD syntax; @context terms must define ' + 'an @id.', 'jsonld.SyntaxError', { + 'context': local_ctx, + 'term': term + }, code='invalid IRI mapping') + # prepend vocab to term + mapping['@id'] = active_ctx['@vocab'] + term + + # IRI mapping now defined + defined[term] = True + + if '@type' in value: + type_ = value['@type'] + if not _is_string(type_): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @type value must be ' + 'a string.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid type mapping') + if type_ != '@id' and type_ != '@vocab': + # expand @type to full IRI + type_ = self._expand_iri( + active_ctx, type_, vocab=True, + local_ctx=local_ctx, defined=defined) + if not _is_absolute_iri(type_): + raise JsonLdError( + 'Invalid JSON-LD syntax; an @context @type value must ' + 'be an absolute IRI.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid type mapping') + if type_.startswith('_:'): + raise JsonLdError( + 'Invalid JSON-LD syntax; an @context @type values ' + 'must be an IRI, not a blank node identifier.', + 'jsonld.SyntaxError', {'context': local_ctx}, + code='invalid type mapping') + # add @type to mapping + mapping['@type'] = type_ + + if '@container' in value: + container = value['@container'] + if container not in ['@list', '@set', '@index', '@language']: + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @container value ' + 'must be one of the following: @list, @set, @index, or ' + '@language.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid container mapping') + if (mapping['reverse'] and container != '@index' and + container != '@set' and container is not None): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @container value for ' + 'an @reverse type definition must be @index or @set.', + 'jsonld.SyntaxError', {'context': local_ctx}, + code='invalid reverse property') + + # add @container to mapping + mapping['@container'] = container + + if '@language' in value and '@type' not in value: + language = value['@language'] + if not (language is None or _is_string(language)): + raise JsonLdError( + 'Invalid JSON-LD syntax; @context @language value must be ' + 'a string or null.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid language mapping') + # add @language to mapping + if language is not None: + language = language.lower() + mapping['@language'] = language + + # disallow aliasing @context and @preserve + id_ = mapping['@id'] + if id_ == '@context' or id_ == '@preserve': + raise JsonLdError( + 'Invalid JSON-LD syntax; @context and @preserve ' + 'cannot be aliased.', 'jsonld.SyntaxError', + {'context': local_ctx}, code='invalid keyword alias') + + def _expand_iri( + self, active_ctx, value, base=False, vocab=False, + local_ctx=None, defined=None): + """ + Expands a string value to a full IRI. The string may be a term, a + prefix, a relative IRI, or an absolute IRI. The associated absolute + IRI will be returned. + + :param active_ctx: the current active context. + :param value: the string value to expand. + :param base: True to resolve IRIs against the base IRI, False not to. + :param vocab: True to concatenate after @vocab, False not to. + :param local_ctx: the local context being processed (only given if + called during context processing). + :param defined: a map for tracking cycles in context definitions (only + given if called during context processing). + + :return: the expanded value. + """ + # already expanded + if value is None or _is_keyword(value): + return value + + # define dependency not if defined + if (local_ctx and value in local_ctx and + defined.get(value) is not True): + self._create_term_definition(active_ctx, local_ctx, value, defined) + + if vocab and value in active_ctx['mappings']: + mapping = active_ctx['mappings'].get(value) + # value is explicitly ignored with None mapping + if mapping is None: + return None + # value is a term + return mapping['@id'] + + # split value into prefix:suffix + if ':' in value: + prefix, suffix = value.split(':', 1) + + # do not expand blank nodes (prefix of '_') or already-absolute + # IRIs (suffix of '//') + if prefix == '_' or suffix.startswith('//'): + return value + + # prefix dependency not defined, define it + if local_ctx and prefix in local_ctx: + self._create_term_definition( + active_ctx, local_ctx, prefix, defined) + + # use mapping if prefix is defined + mapping = active_ctx['mappings'].get(prefix) + if mapping: + return mapping['@id'] + suffix + + # already absolute IRI + return value + + # prepend vocab + if vocab and '@vocab' in active_ctx: + return active_ctx['@vocab'] + value + + # resolve against base + rval = value + if base: + rval = prepend_base(active_ctx['@base'], rval) + + return rval + + def _find_context_urls(self, input_, urls, replace, base): + """ + Finds all @context URLs in the given JSON-LD input. + + :param input_: the JSON-LD input. + :param urls: a map of URLs (url => False/@contexts). + :param replace: True to replace the URLs in the given input with + the @contexts from the urls map, False not to. + :param base: the base URL to resolve relative URLs against. + """ + if _is_array(input_): + for e in input_: + self._find_context_urls(e, urls, replace, base) + elif _is_object(input_): + for k, v in input_.items(): + if k != '@context': + self._find_context_urls(v, urls, replace, base) + continue + + # array @context + if _is_array(v): + length = len(v) + for i in range(length): + if _is_string(v[i]): + url = prepend_base(base, v[i]) + # replace w/@context if requested + if replace: + ctx = urls[url] + if _is_array(ctx): + # add flattened context + v.pop(i) + for e in reversed(ctx): + v.insert(i, e) + i += len(ctx) - 1 + length = len(v) + else: + v[i] = ctx + # @context URL found + elif url not in urls: + urls[url] = False + # string @context + elif _is_string(v): + v = prepend_base(base, v) + # replace w/@context if requested + if replace: + input_[k] = urls[v] + # @context URL found + elif v not in urls: + urls[v] = False + + def _retrieve_context_urls(self, input_, cycles, load_document, base=''): + """ + Retrieves external @context URLs using the given document loader. Each + instance of @context in the input that refers to a URL will be + replaced with the JSON @context found at that URL. + + :param input_: the JSON-LD input with possible contexts. + :param cycles: an object for tracking context cycles. + :param load_document(url): the document loader. + :param base: the base URL to resolve relative URLs against. + + :return: the result. + """ + if len(cycles) > MAX_CONTEXT_URLS: + raise JsonLdError( + 'Maximum number of @context URLs exceeded.', + 'jsonld.ContextUrlError', {'max': MAX_CONTEXT_URLS}, + code='loading remote context failed') + + # for tracking URLs to retrieve + urls = {} + + # find all URLs in the given input + self._find_context_urls(input_, urls, replace=False, base=base) + + # queue all unretrieved URLs + queue = [] + for url, ctx in urls.items(): + if ctx is False: + queue.append(url) + + # retrieve URLs in queue + for url in queue: + # check for context URL cycle + if url in cycles: + raise JsonLdError( + 'Cyclical @context URLs detected.', + 'jsonld.ContextUrlError', {'url': url}, + code='recursive context inclusion') + cycles_ = copy.deepcopy(cycles) + cycles_[url] = True + + # retrieve URL + try: + remote_doc = load_document(url) + ctx = remote_doc['document'] + except Exception as cause: + raise JsonLdError( + 'Dereferencing a URL did not result in a valid JSON-LD ' + 'context.', + 'jsonld.ContextUrlError', {'url': url}, + code='loading remote context failed', cause=cause) + + # parse string context as JSON + if _is_string(ctx): + try: + ctx = json.loads(ctx) + except Exception as cause: + raise JsonLdError( + 'Could not parse JSON from URL.', + 'jsonld.ParseError', {'url': url}, + code='loading remote context failed', cause=cause) + + # ensure ctx is an object + if not _is_object(ctx): + raise JsonLdError( + 'Dereferencing a URL did not result in a valid JSON-LD ' + 'object.', + 'jsonld.InvalidUrl', {'url': url}, + code='invalid remote context') + + # use empty context if no @context key is present + if '@context' not in ctx: + ctx = {'@context': {}} + else: + ctx = {'@context': ctx['@context']} + + # append context URL to context if given + if remote_doc['contextUrl'] is not None: + ctx['@context'] = JsonLdProcessor.arrayify(ctx['@context']) + ctx['@context'].append(remote_doc['contextUrl']) + + # recurse + self._retrieve_context_urls(ctx, cycles_, load_document, url) + urls[url] = ctx['@context'] + + # replace all URLs in the input + self._find_context_urls(input_, urls, replace=True, base=base) + + def _get_initial_context(self, options): + """ + Gets the initial context. + + :param options: the options to use. + [base] the document base IRI. + + :return: the initial context. + """ + return { + '@base': options['base'], + 'mappings': {}, + 'inverse': None + } + + def _get_inverse_context(self, active_ctx): + """ + Generates an inverse context for use in the compaction algorithm, if + not already generated for the given active context. + + :param active_ctx: the active context to use. + + :return: the inverse context. + """ + # inverse context already generated + if active_ctx['inverse']: + return active_ctx['inverse'] + + inverse = active_ctx['inverse'] = {} + + # handle default language + default_language = active_ctx.get('@language', '@none') + + # create term selections for each mapping in the context, ordered by + # shortest and then lexicographically least + for term, mapping in sorted( + active_ctx['mappings'].items(), + key=cmp_to_key(_compare_shortest_least)): + if mapping is None: + continue + + # add term selection where it applies + container = mapping.get('@container', '@none') + + # iterate over every IRI in the mapping + iris = JsonLdProcessor.arrayify(mapping['@id']) + for iri in iris: + container_map = inverse.setdefault(iri, {}) + entry = container_map.setdefault( + container, {'@language': {}, '@type': {}}) + + # term is preferred for values using @reverse + if mapping['reverse']: + entry['@type'].setdefault('@reverse', term) + # term is preferred for values using specific type + elif '@type' in mapping: + entry['@type'].setdefault(mapping['@type'], term) + # term is preferred for values using specific language + elif '@language' in mapping: + language = mapping['@language'] + if language is None: + language = '@null' + entry['@language'].setdefault(language, term) + # term is preferred for values w/default language or not type + # and no language + else: + # add an entry for the default language + entry['@language'].setdefault(default_language, term) + # add entries for no type and no language + entry['@type'].setdefault('@none', term) + entry['@language'].setdefault('@none', term) + + return inverse + + def _clone_active_context(self, active_ctx): + """ + Clones an active context, creating a child active context. + + :param active_ctx: the active context to clone. + + :return: a clone (child) of the active context. + """ + child = { + '@base': active_ctx['@base'], + 'mappings': copy.deepcopy(active_ctx['mappings']), + 'inverse': None + } + if '@language' in active_ctx: + child['@language'] = active_ctx['@language'] + if '@vocab' in active_ctx: + child['@vocab'] = active_ctx['@vocab'] + return child + + +# register the N-Quads RDF parser +register_rdf_parser('application/nquads', JsonLdProcessor.parse_nquads) + + +class JsonLdError(Exception): + """ + Base class for JSON-LD errors. + """ + + def __init__(self, message, type_, details=None, code=None, cause=None): + Exception.__init__(self, message) + self.type = type_ + self.details = details + self.code = code + self.cause = cause + self.causeTrace = traceback.extract_tb(*sys.exc_info()[2:]) + + def __str__(self): + rval = repr(self.message) + rval += '\nType: ' + self.type + if self.code: + rval += '\nCode: ' + self.code + if self.details: + rval += '\nDetails: ' + repr(self.details) + if self.cause: + rval += '\nCause: ' + str(self.cause) + rval += ''.join(traceback.format_list(self.causeTrace)) + return rval + + +class UniqueNamer(object): + """ + A UniqueNamer issues unique names, keeping track of any previously issued + names. + """ + + def __init__(self, prefix): + """ + Initializes a new UniqueNamer. + + :param prefix: the prefix to use (''). + """ + self.prefix = prefix + self.counter = 0 + self.existing = {} + self.order = [] + + """ + Gets the new name for the given old name, where if no old name is + given a new name will be generated. + + :param [old_name]: the old name to get the new name for. + + :return: the new name. + """ + def get_name(self, old_name=None): + # return existing old name + if old_name and old_name in self.existing: + return self.existing[old_name] + + # get next name + name = self.prefix + str(self.counter) + self.counter += 1 + + # save mapping + if old_name is not None: + self.existing[old_name] = name + self.order.append(old_name) + + return name + + def is_named(self, old_name): + """ + Returns True if the given old name has already been assigned a new + name. + + :param old_name: the old name to check. + + :return: True if the old name has been assigned a new name, False if + not. + """ + return old_name in self.existing + + +def permutations(elements): + """ + Generates all of the possible permutations for the given list of elements. + + :param elements: the list of elements to permutate. + """ + # begin with sorted elements + elements.sort() + # initialize directional info for permutation algorithm + left = {} + for v in elements: + left[v] = True + + length = len(elements) + last = length - 1 + while True: + yield elements + + # Calculate the next permutation using the Steinhaus-Johnson-Trotter + # permutation algorithm. + + # get largest mobile element k + # (mobile: element is greater than the one it is looking at) + k, pos = None, 0 + for i in range(length): + e = elements[i] + is_left = left[e] + if((k is None or e > k) and + ((is_left and i > 0 and e > elements[i - 1]) or + (not is_left and i < last and e > elements[i + 1]))): + k, pos = e, i + + # no more permutations + if k is None: + raise StopIteration + + # swap k and the element it is looking at + swap = pos - 1 if left[k] else pos + 1 + elements[pos], elements[swap] = elements[swap], k + + # reverse the direction of all elements larger than k + for i in range(length): + if elements[i] > k: + left[elements[i]] = not left[elements[i]] + + +def _compare_shortest_least(a, b): + """ + Compares two strings first based on length and then lexicographically. + + :param a: the first string. + :param b: the second string. + + :return: -1 if a < b, 1 if a > b, 0 if a == b. + """ + rval = cmp(len(a), len(b)) + if rval == 0: + rval = cmp(a, b) + return rval + + +def _is_keyword(v): + """ + Returns whether or not the given value is a keyword. + + :param v: the value to check. + + :return: True if the value is a keyword, False if not. + """ + if not _is_string(v): + return False + return v in KEYWORDS + + +def _is_object(v): + """ + Returns True if the given value is an Object. + + :param v: the value to check. + + :return: True if the value is an Object, False if not. + """ + return isinstance(v, dict) + + +def _is_empty_object(v): + """ + Returns True if the given value is an empty Object. + + :param v: the value to check. + + :return: True if the value is an empty Object, False if not. + """ + return _is_object(v) and len(v) == 0 + + +def _is_array(v): + """ + Returns True if the given value is an Array. + + :param v: the value to check. + + :return: True if the value is an Array, False if not. + """ + return isinstance(v, list) + + +def _is_string(v): + """ + Returns True if the given value is a String. + + :param v: the value to check. + + :return: True if the value is a String, False if not. + """ + return isinstance(v, basestring) + + +def _validate_type_value(v): + """ + Raises an exception if the given value is not a valid @type value. + + :param v: the value to check. + """ + # must be a string or empty object + if (_is_string(v) or _is_empty_object(v)): + return + + # must be an array + is_valid = False + if _is_array(v): + # must contain only strings + is_valid = True + for e in v: + if not _is_string(e): + is_valid = False + break + + if not is_valid: + raise JsonLdError( + 'Invalid JSON-LD syntax; "@type" value must a string, an array of ' + 'strings, or an empty object.', + 'jsonld.SyntaxError', {'value': v}, code='invalid type value') + + +def _is_bool(v): + """ + Returns True if the given value is a Boolean. + + :param v: the value to check. + + :return: True if the value is a Boolean, False if not. + """ + return isinstance(v, bool) + + +def _is_integer(v): + """ + Returns True if the given value is an Integer. + + :param v: the value to check. + + :return: True if the value is an Integer, False if not. + """ + return isinstance(v, Integral) + + +def _is_double(v): + """ + Returns True if the given value is a Double. + + :param v: the value to check. + + :return: True if the value is a Double, False if not. + """ + return not isinstance(v, Integral) and isinstance(v, Real) + + +def _is_numeric(v): + """ + Returns True if the given value is numeric. + + :param v: the value to check. + + :return: True if the value is numeric, False if not. + """ + try: + float(v) + return True + except ValueError: + return False + + +def _is_subject(v): + """ + Returns True if the given value is a subject with properties. + + :param v: the value to check. + + :return: True if the value is a subject with properties, False if not. + """ + # Note: A value is a subject if all of these hold True: + # 1. It is an Object. + # 2. It is not a @value, @set, or @list. + # 3. It has more than 1 key OR any existing key is not @id. + rval = False + if (_is_object(v) and + '@value' not in v and '@set' not in v and '@list' not in v): + rval = len(v) > 1 or '@id' not in v + return rval + + +def _is_subject_reference(v): + """ + Returns True if the given value is a subject reference. + + :param v: the value to check. + + :return: True if the value is a subject reference, False if not. + """ + # Note: A value is a subject reference if all of these hold True: + # 1. It is an Object. + # 2. It has a single key: @id. + return (_is_object(v) and len(v) == 1 and '@id' in v) + + +def _is_value(v): + """ + Returns True if the given value is a @value. + + :param v: the value to check. + + :return: True if the value is a @value, False if not. + """ + # Note: A value is a @value if all of these hold True: + # 1. It is an Object. + # 2. It has the @value property. + return _is_object(v) and '@value' in v + + +def _is_list(v): + """ + Returns True if the given value is a @list. + + :param v: the value to check. + + :return: True if the value is a @list, False if not. + """ + # Note: A value is a @list if all of these hold True: + # 1. It is an Object. + # 2. It has the @list property. + return _is_object(v) and '@list' in v + + +def _is_bnode(v): + """ + Returns True if the given value is a blank node. + + :param v: the value to check. + + :return: True if the value is a blank node, False if not. + """ + # Note: A value is a blank node if all of these hold True: + # 1. It is an Object. + # 2. If it has an @id key its value begins with '_:'. + # 3. It has no keys OR is not a @value, @set, or @list. + rval = False + if _is_object(v): + if '@id' in v: + rval = v['@id'].startswith('_:') + else: + rval = (len(v) == 0 or not + ('@value' in v or '@set' in v or '@list' in v)) + return rval + + +def _is_absolute_iri(v): + """ + Returns True if the given value is an absolute IRI, False if not. + + :param v: the value to check. + + :return: True if the value is an absolute IRI, False if not. + """ + return ':' in v + + +class ActiveContextCache(object): + """ + An ActiveContextCache caches active contexts so they can be reused without + the overhead of recomputing them. + """ + + def __init__(self, size=100): + self.order = deque() + self.cache = {} + self.size = size + + def get(self, active_ctx, local_ctx): + key1 = json.dumps(active_ctx) + key2 = json.dumps(local_ctx) + return self.cache.get(key1, {}).get(key2) + + def set(self, active_ctx, local_ctx, result): + if len(self.order) == self.size: + entry = self.order.popleft() + del self.cache[entry['activeCtx']][entry['localCtx']] + key1 = json.dumps(active_ctx) + key2 = json.dumps(local_ctx) + self.order.append({'activeCtx': key1, 'localCtx': key2}) + self.cache.setdefault(key1, {})[key2] = json.loads(json.dumps(result)) + + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Used to verify SSL certificates when resolving URLs. + Taken from: http://thejosephturner.com/blog/2011/03/19/https-\ + certificate-verification-in-python-with-urllib2/ + """ + + def connect(self): + global _trust_root_certificates + # overrides the version in httplib to do certificate verification + sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() + # wrap the socket using verification with trusted_root_certs + self.sock = ssl.wrap_socket(sock, + self.key_file, + self.cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=_trust_root_certificates) + + +class VerifiedHTTPSHandler(HTTPSHandler): + """ + Wraps urllib2 HTTPS connections enabling SSL certificate verification. + """ + + def __init__(self, connection_class=VerifiedHTTPSConnection): + self.specialized_conn_class = connection_class + HTTPSHandler.__init__(self) + + def https_open(self, req): + return self.do_open(self.specialized_conn_class, req) + + +# the path to the system's default trusted root SSL certificates +_trust_root_certificates = None +_possible_trust_root_certificates = [ + '/etc/ssl/certs/ca-certificates.crt', + '~/Library/OpenSSL/certs/ca-certificates.crt', + '/System/Library/OpenSSL/certs/ca-certificates.crt', +] +for path in _possible_trust_root_certificates: + path = os.path.expanduser(path) + if os.path.exists(path): + _trust_root_certificates = path + break +# FIXME: warn if not found? MacOS X uses keychain vs file. + + +# Shared in-memory caches. +_cache = { + 'activeCtx': ActiveContextCache() +} diff --git a/roles.py b/roles.py index 9cfa1cc7a..fe60b2ab3 100644 --- a/roles.py +++ b/roles.py @@ -37,6 +37,26 @@ def clearModeratorStatus(baseDir: str) -> None: saveJson(actorJson, filename) +def clearEditorStatus(baseDir: str) -> None: + """Removes editor status from all accounts + This could be slow if there are many users, but only happens + rarely when editors are appointed or removed + """ + directory = os.fsencode(baseDir + '/accounts/') + for f in os.scandir(directory): + f = f.name + filename = os.fsdecode(f) + if filename.endswith(".json") and '@' in filename: + filename = os.path.join(baseDir + '/accounts/', filename) + if '"editor"' in open(filename).read(): + actorJson = loadJson(filename) + if actorJson: + if actorJson['roles'].get('instance'): + if 'editor' in actorJson['roles']['instance']: + actorJson['roles']['instance'].remove('editor') + saveJson(actorJson, filename) + + def addModerator(baseDir: str, nickname: str, domain: str) -> None: """Adds a moderator nickname to the file """ diff --git a/scripts/clearnewswire b/scripts/clearnewswire new file mode 100755 index 000000000..39999724f --- /dev/null +++ b/scripts/clearnewswire @@ -0,0 +1,10 @@ +#!/bin/bash +rm accounts/news@*/outbox/* +rm accounts/news@*/postcache/* +rm accounts/news@*/outbox.index +if [ -f accounts/.newswirestate.json ]; then + rm accounts/.newswirestate.json +fi +if [ -f accounts/.currentnewswire.json ]; then + rm accounts/.currentnewswire.json +fi diff --git a/tests.py b/tests.py index cc0a472c8..1f1bc86d6 100644 --- a/tests.py +++ b/tests.py @@ -78,6 +78,7 @@ from content import addHtmlTags from content import removeLongWords from content import replaceContentDuplicates from content import removeTextFormatting +from content import removeHtmlTag from theme import setCSSparam from jsonldsig import testSignJsonld from jsonldsig import jsonldVerify @@ -287,7 +288,8 @@ def createServerAlice(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Alice') - runDaemon(False, False, 5, True, True, 'en', __version__, + runDaemon(False, 0, False, 1, False, False, False, + 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, httpPrefix, federationList, maxMentions, maxEmoji, False, @@ -349,7 +351,8 @@ def createServerBob(path: str, domain: str, port: int, onionDomain = None i2pDomain = None print('Server running: Bob') - runDaemon(False, False, 5, True, True, 'en', __version__, + runDaemon(False, 0, False, 1, False, False, False, + 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, httpPrefix, federationList, maxMentions, maxEmoji, False, @@ -385,7 +388,8 @@ def createServerEve(path: str, domain: str, port: int, federationList: [], onionDomain = None i2pDomain = None print('Server running: Eve') - runDaemon(False, False, 5, True, True, 'en', __version__, + runDaemon(False, 0, False, 1, False, False, False, + 5, True, True, 'en', __version__, "instanceId", False, path, domain, onionDomain, i2pDomain, None, port, port, httpPrefix, federationList, maxMentions, maxEmoji, False, @@ -1766,9 +1770,9 @@ def testGetStatusNumber(): prevStatusNumber = int(statusNumber) -def testCommentJson() -> None: - print('testCommentJson') - filename = '/tmp/test.json' +def testJsonString() -> None: + print('testJsonString') + filename = '.epicyon_tests_testJsonString.json' messageStr = "Crème brûlée यह एक परीक्षण ह" testJson = { "content": messageStr @@ -1779,6 +1783,7 @@ def testCommentJson() -> None: assert receivedJson['content'] == messageStr encodedStr = json.dumps(testJson, ensure_ascii=False) assert messageStr in encodedStr + os.remove(filename) def testSaveLoadJson(): @@ -1787,7 +1792,7 @@ def testSaveLoadJson(): "param1": 3, "param2": '"Crème brûlée यह एक परीक्षण ह"' } - testFilename = '/tmp/.epicyonTestSaveLoadJson.json' + testFilename = '.epicyon_tests_testSaveLoadJson.json' if os.path.isfile(testFilename): os.remove(testFilename) assert saveJson(testJson, testFilename) @@ -2159,8 +2164,18 @@ def testReplaceEmailQuote(): assert resultStr == expectedStr +def testRemoveHtmlTag(): + print('testRemoveHtmlTag') + testStr = "

" + resultStr = removeHtmlTag(testStr, 'width') + assert resultStr == "

" + + def runAllTests(): print('Running tests...') + testRemoveHtmlTag() testReplaceEmailQuote() testConstantTimeStringCheck() testTranslations() @@ -2177,7 +2192,7 @@ def runAllTests(): testRecentPostsCache() testTheme() testSaveLoadJson() - testCommentJson() + testJsonString() testGetStatusNumber() testAddEmoji() testActorParsing() diff --git a/theme.py b/theme.py index ee64481c6..e236af711 100644 --- a/theme.py +++ b/theme.py @@ -254,6 +254,16 @@ def setThemeIndymedia(baseDir: str): "search": "jpg" } themeParams = { + "font-size-newswire": "18px", + "font-size-newswire-mobile": "48px", + "line-spacing-newswire": "100%", + "newswire-item-moderated-color": "white", + "newswire-date-moderated-color": "white", + "newswire-date-color": "white", + "newswire-voted-background-color": "black", + "column-left-image-width-mobile": "40vw", + "column-right-fg-color": "#ff9900", + "column-right-fg-color-voted-on": "red", "button-corner-radius": "5px", "timeline-border-radius": "5px", "focus-color": "blue", @@ -996,7 +1006,7 @@ def setThemeImages(baseDir: str, name: str) -> None: pass -def setTheme(baseDir: str, name: str) -> bool: +def setTheme(baseDir: str, name: str, domain: str) -> bool: result = False prevThemeName = getTheme(baseDir) @@ -1019,6 +1029,15 @@ def setTheme(baseDir: str, name: str) -> bool: result = True setCustomFont(baseDir) + + # set the news avatar + newsAvatarThemeFilename = \ + baseDir + '/img/icons/' + name + '/avatar_news.png' + if os.path.isfile(newsAvatarThemeFilename): + newsAvatarFilename = \ + baseDir + '/accounts/news@' + domain + '/avatar.png' + copyfile(newsAvatarThemeFilename, newsAvatarFilename) + grayscaleFilename = baseDir + '/accounts/.grayscale' if os.path.isfile(grayscaleFilename): enableGrayscale(baseDir) diff --git a/translations/ar.json b/translations/ar.json index 096198c27..ca37a9949 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -297,5 +297,16 @@ "Edit newswire": "تحرير الأخبار", "Add RSS feed links below.": "إضافة روابط تغذية RSS أدناه.", "Newswire RSS Feed": "Newswire موجز RSS", - "Nicknames whose blog entries appear on the newswire.": "الألقاب التي تظهر إدخالات المدونة الخاصة بها على موقع الأخبار." + "Nicknames whose blog entries appear on the newswire.": "الألقاب التي تظهر إدخالات المدونة الخاصة بها على موقع الأخبار.", + "Posts to be approved": "الوظائف المطلوب الموافقة عليها", + "Discuss": "مناقشة", + "Moderator Discussion": "مناقشة المنسق", + "Vote": "تصويت", + "Remove Vote": "إزالة التصويت", + "This is a news instance": "هذا مثال أخبار", + "News": "أخبار", + "Read more...": "اقرأ أكثر...", + "Edit News Post": "تحرير منشور الأخبار", + "A list of editor nicknames. One per line.": "قائمة بأسماء المحرر. واحد في كل سطر.", + "Site Editors": "محررو الموقع" } diff --git a/translations/ca.json b/translations/ca.json index 0e2fb495e..78a90aff9 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -297,5 +297,16 @@ "Edit newswire": "Editeu newswire", "Add RSS feed links below.": "Afegiu enllaços de canals RSS a continuació.", "Newswire RSS Feed": "Feed RSS de Newswire", - "Nicknames whose blog entries appear on the newswire.": "Sobrenoms les entrades del bloc apareixen a newswire." + "Nicknames whose blog entries appear on the newswire.": "Sobrenoms les entrades del bloc apareixen a newswire.", + "Posts to be approved": "Missatges per aprovar", + "Discuss": "Discuteix", + "Moderator Discussion": "Discussió sobre moderadors", + "Vote": "Notícies", + "Remove Vote": "Elimina el vot", + "This is a news instance": "Aquesta és una instància de notícies", + "News": "Notícies", + "Read more...": "Llegeix més...", + "Edit News Post": "Edita la publicació de notícies", + "A list of editor nicknames. One per line.": "Una llista de sobrenoms de l'editor. Un per línia.", + "Site Editors": "Editors de llocs" } diff --git a/translations/cy.json b/translations/cy.json index 8c0b0192a..a68b2456f 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -297,5 +297,16 @@ "Edit newswire": "Golygu newyddion", "Add RSS feed links below.": "Ychwanegwch ddolenni porthiant RSS isod.", "Newswire RSS Feed": "Newswire RSS Feed", - "Nicknames whose blog entries appear on the newswire.": "Llysenwau y mae eu cofnodion blog yn ymddangos ar y we newyddion." + "Nicknames whose blog entries appear on the newswire.": "Llysenwau y mae eu cofnodion blog yn ymddangos ar y we newyddion.", + "Posts to be approved": "Swyddi i'w cymeradwyo", + "Discuss": "Trafodwch", + "Moderator Discussion": "Trafodaeth Cymedrolwr", + "Vote": "Newyddion", + "Remove Vote": "Tynnwch y Bleidlais", + "This is a news instance": "Dyma enghraifft newyddion", + "News": "Newyddion", + "Read more...": "Darllen mwy...", + "Edit News Post": "Golygu News News", + "A list of editor nicknames. One per line.": "Rhestr o lysenwau golygydd. Un i bob llinell.", + "Site Editors": "Golygyddion Safle" } diff --git a/translations/de.json b/translations/de.json index b428e1a72..06bdd7e98 100644 --- a/translations/de.json +++ b/translations/de.json @@ -297,5 +297,16 @@ "Edit newswire": "Newswire bearbeiten", "Add RSS feed links below.": "Fügen Sie unten RSS-Feed-Links hinzu.", "Newswire RSS Feed": "Newswire RSS Feed", - "Nicknames whose blog entries appear on the newswire.": "Spitznamen, deren Blogeinträge im Newswire erscheinen." + "Nicknames whose blog entries appear on the newswire.": "Spitznamen, deren Blogeinträge im Newswire erscheinen.", + "Posts to be approved": "Zu genehmigende Beiträge", + "Discuss": "Diskutieren", + "Moderator Discussion": "Moderatorendiskussion", + "Vote": "Abstimmung", + "Remove Vote": "Abstimmung entfernen", + "This is a news instance": "Dies ist eine Nachrichteninstanz", + "News": "Nachrichten", + "Read more...": "Weiterlesen...", + "Edit News Post": "Nachrichtenbeitrag bearbeiten", + "A list of editor nicknames. One per line.": "Eine Liste der Editor-Spitznamen. Eine pro Zeile.", + "Site Editors": "Site-Editoren" } diff --git a/translations/en.json b/translations/en.json index 644161721..9a10f674f 100644 --- a/translations/en.json +++ b/translations/en.json @@ -212,8 +212,8 @@ "Remove Twitter posts": "Remove Twitter posts", "Sensitive": "Sensitive", "Word Replacements": "Word Replacements", - "Happening Today": "Happening Today", - "Happening This Week": "Happening This Week", + "Happening Today": "Today", + "Happening This Week": "This Week", "Blog": "Blog", "Blogs": "Blogs", "Title": "Title", @@ -295,7 +295,18 @@ "Right column image": "Right column image", "RSS feed for this site": "RSS feed for this site", "Edit newswire": "Edit newswire", - "Add RSS feed links below.": "Add RSS feed links below.", + "Add RSS feed links below.": "RSS feed links below. Add a * at the beginning or end to indicate that a feed should be moderated.", "Newswire RSS Feed": "Newswire RSS Feed", - "Nicknames whose blog entries appear on the newswire.": "Nicknames whose blog entries appear on the newswire." + "Nicknames whose blog entries appear on the newswire.": "Nicknames whose blog entries appear on the newswire.", + "Posts to be approved": "Posts to be approved", + "Discuss": "Discuss", + "Moderator Discussion": "Moderator Discussion", + "Vote": "Vote", + "Remove Vote": "Remove Vote", + "This is a news instance": "This is a news instance", + "News": "News", + "Read more...": "Read more...", + "Edit News Post": "Edit News Post", + "A list of editor nicknames. One per line.": "A list of editor nicknames. One per line.", + "Site Editors": "Site Editors" } diff --git a/translations/es.json b/translations/es.json index 457acde58..6970b49bd 100644 --- a/translations/es.json +++ b/translations/es.json @@ -297,5 +297,16 @@ "Edit newswire": "Editar newswire", "Add RSS feed links below.": "Agregue los enlaces de fuentes RSS a continuación.", "Newswire RSS Feed": "Canal RSS de Newswire", - "Nicknames whose blog entries appear on the newswire.": "Apodos cuyas entradas de blog aparecen en el newswire." + "Nicknames whose blog entries appear on the newswire.": "Apodos cuyas entradas de blog aparecen en el newswire.", + "Posts to be approved": "Publicaciones a aprobar", + "Discuss": "Discutir", + "Moderator Discussion": "Discusión del moderador", + "Vote": "Votar", + "Remove Vote": "Eliminar voto", + "This is a news instance": "Esta es una instancia de noticias", + "News": "Noticias", + "Read more...": "Lee mas...", + "Edit News Post": "Editar publicación de noticias", + "A list of editor nicknames. One per line.": "Una lista de apodos de los editores. Uno por línea.", + "Site Editors": "Editores del sitio" } diff --git a/translations/fr.json b/translations/fr.json index b9b0c00e2..65d770121 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -297,5 +297,16 @@ "Edit newswire": "Modifier le fil d'actualité", "Add RSS feed links below.": "Ajoutez des liens de flux RSS ci-dessous.", "Newswire RSS Feed": "Flux RSS de Newswire", - "Nicknames whose blog entries appear on the newswire.": "Surnoms dont les entrées de blog apparaissent sur le fil de presse." + "Nicknames whose blog entries appear on the newswire.": "Surnoms dont les entrées de blog apparaissent sur le fil de presse.", + "Posts to be approved": "Postes à approuver", + "Discuss": "Discuter", + "Moderator Discussion": "Discussion du modérateur", + "Vote": "Voter", + "Remove Vote": "Supprimer le vote", + "This is a news instance": "Ceci est une instance d'actualité", + "News": "Nouvelles", + "Read more...": "Lire la suite...", + "Edit News Post": "Modifier l'article d'actualité", + "A list of editor nicknames. One per line.": "Une liste de surnoms d'éditeur. Un par ligne.", + "Site Editors": "Éditeurs du site" } diff --git a/translations/ga.json b/translations/ga.json index 1853bd945..d97ad2671 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -297,5 +297,16 @@ "Edit newswire": "Cuir sreang nuachta in eagar", "Add RSS feed links below.": "Cuir naisc beatha RSS thíos.", "Newswire RSS Feed": "Newswire RSS Feed", - "Nicknames whose blog entries appear on the newswire.": "Leasainmneacha a bhfuil a n-iontrálacha blag le feiceáil ar an sreang nuachta." + "Nicknames whose blog entries appear on the newswire.": "Leasainmneacha a bhfuil a n-iontrálacha blag le feiceáil ar an sreang nuachta.", + "Posts to be approved": "Poist le ceadú", + "Discuss": "Pléigh", + "Moderator Discussion": "Plé Modhnóir", + "Vote": "Vóta", + "Remove Vote": "Bain Vóta", + "This is a news instance": "Is sampla nuachta é seo", + "News": "Nuacht", + "Read more...": "Leigh Nios mo...", + "Edit News Post": "Cuir News Post in eagar", + "A list of editor nicknames. One per line.": "Liosta leasainmneacha eagarthóra. Ceann in aghaidh na líne.", + "Site Editors": "Eagarthóirí Suímh" } diff --git a/translations/hi.json b/translations/hi.json index 45f6af94a..f1ed564bb 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -297,5 +297,16 @@ "Edit newswire": "नवांश संपादित करें", "Add RSS feed links below.": "नीचे आरएसएस फ़ीड लिंक जोड़ें।", "Newswire RSS Feed": "Newswire RSS फ़ीड", - "Nicknames whose blog entries appear on the newswire.": "उपनाम जिनकी ब्लॉग प्रविष्टियाँ न्यूज़वायर पर दिखाई देती हैं।" + "Nicknames whose blog entries appear on the newswire.": "उपनाम जिनकी ब्लॉग प्रविष्टियाँ न्यूज़वायर पर दिखाई देती हैं।", + "Posts to be approved": "स्वीकृत किए जाने वाले पद", + "Discuss": "चर्चा करें", + "Moderator Discussion": "मॉडरेटर चर्चा", + "Vote": "वोट", + "Remove Vote": "वोट हटा दें", + "This is a news instance": "यह एक समाचार का उदाहरण है", + "News": "समाचार", + "Read more...": "अधिक पढ़ें...", + "Edit News Post": "समाचार पोस्ट संपादित करें", + "A list of editor nicknames. One per line.": "संपादक उपनामों की एक सूची। प्रति पंक्ति एक।", + "Site Editors": "साइट संपादकों" } diff --git a/translations/it.json b/translations/it.json index eba3c33fe..d01e1fd46 100644 --- a/translations/it.json +++ b/translations/it.json @@ -297,5 +297,16 @@ "Edit newswire": "Modifica newswire", "Add RSS feed links below.": "Aggiungi i link ai feed RSS di seguito.", "Newswire RSS Feed": "Feed RSS di Newswire", - "Nicknames whose blog entries appear on the newswire.": "Soprannomi le cui voci di blog compaiono nel newswire." + "Nicknames whose blog entries appear on the newswire.": "Soprannomi le cui voci di blog compaiono nel newswire.", + "Posts to be approved": "Post da approvare", + "Discuss": "Discutere", + "Moderator Discussion": "Discussione del moderatore", + "Vote": "Votazione", + "Remove Vote": "Rimuovi voto", + "This is a news instance": "Questa è un'istanza di notizie", + "News": "Notizia", + "Read more...": "Leggi di più...", + "Edit News Post": "Modifica post di notizie", + "A list of editor nicknames. One per line.": "Un elenco di soprannomi dell'editor. Uno per riga.", + "Site Editors": "Editori del sito" } diff --git a/translations/ja.json b/translations/ja.json index 4b9a9cbb4..1867a618d 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -297,5 +297,16 @@ "Edit newswire": "ニュースワイヤーを編集", "Add RSS feed links below.": "以下にRSSフィードリンクを追加します。", "Newswire RSS Feed": "NewswireRSSフィード", - "Nicknames whose blog entries appear on the newswire.": "ブログエントリがニュースワイヤーに表示されるニックネーム。" + "Nicknames whose blog entries appear on the newswire.": "ブログエントリがニュースワイヤーに表示されるニックネーム。", + "Posts to be approved": "承認される投稿", + "Discuss": "議論する", + "Moderator Discussion": "モデレーターディスカッション", + "Vote": "投票", + "Remove Vote": "投票を削除", + "This is a news instance": "これはニュースインスタンスです", + "News": "ニュース", + "Read more...": "続きを読む...", + "Edit News Post": "ニュース投稿を編集する", + "A list of editor nicknames. One per line.": "編集者のニックネームのリスト。 1行に1つ。", + "Site Editors": "サイト編集者" } diff --git a/translations/oc.json b/translations/oc.json index 6f39c0ae5..5f54ec75d 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -291,7 +291,18 @@ "Right column image": "Right column image", "RSS feed for this site": "RSS feed for this site", "Edit newswire": "Edit newswire", - "Add RSS feed links below.": "Add RSS feed links below.", + "Add RSS feed links below.": "RSS feed links below. Add a * at the beginning or end to indicate that a feed should be moderated.", "Newswire RSS Feed": "Newswire RSS Feed", - "Nicknames whose blog entries appear on the newswire.": "Nicknames whose blog entries appear on the newswire." + "Nicknames whose blog entries appear on the newswire.": "Nicknames whose blog entries appear on the newswire.", + "Posts to be approved": "Posts to be approved", + "Discuss": "Discuss", + "Moderator Discussion": "Moderator Discussion", + "Vote": "Vote", + "Remove Vote": "Remove Vote", + "This is a news instance": "This is a news instance", + "News": "News", + "Read more...": "Read more...", + "Edit News Post": "Edit News Post", + "A list of editor nicknames. One per line.": "A list of editor nicknames. One per line.", + "Site Editors": "Site Editors" } diff --git a/translations/pt.json b/translations/pt.json index af16daa72..78832177e 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -297,5 +297,16 @@ "Edit newswire": "Editar notícias", "Add RSS feed links below.": "Adicione links de feed RSS abaixo.", "Newswire RSS Feed": "Feed RSS da Newswire", - "Nicknames whose blog entries appear on the newswire.": "Apelidos cujas entradas de blog aparecem nos jornais." + "Nicknames whose blog entries appear on the newswire.": "Apelidos cujas entradas de blog aparecem nos jornais.", + "Posts to be approved": "Postagens a serem aprovadas", + "Discuss": "Discutir", + "Moderator Discussion": "Discussão do moderador", + "Vote": "Voto", + "Remove Vote": "Remover voto", + "This is a news instance": "Esta é uma instância de notícias", + "News": "Notícia", + "Read more...": "Consulte Mais informação...", + "Edit News Post": "Editar Postagem de Notícias", + "A list of editor nicknames. One per line.": "Uma lista de apelidos de editores. Um por linha.", + "Site Editors": "Editores do site" } diff --git a/translations/ru.json b/translations/ru.json index 37f5d6659..798d51b18 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -297,5 +297,16 @@ "Edit newswire": "Редактировать ленту новостей", "Add RSS feed links below.": "Добавьте ссылки на RSS-канал ниже.", "Newswire RSS Feed": "Лента новостей RSS", - "Nicknames whose blog entries appear on the newswire.": "Псевдонимы, чьи записи блога появляются в ленте новостей." + "Nicknames whose blog entries appear on the newswire.": "Псевдонимы, чьи записи блога появляются в ленте новостей.", + "Posts to be approved": "Посты на утверждение", + "Discuss": "Обсудить", + "Moderator Discussion": "Обсуждение модератором", + "Vote": "Голос", + "Remove Vote": "Удалить голос", + "This is a news instance": "Это новостной экземпляр", + "News": "Новости", + "Read more...": "Подробнее...", + "Edit News Post": "Редактировать новость", + "A list of editor nicknames. One per line.": "Список ников редакторов. По одному на строку.", + "Site Editors": "Редакторы сайта" } diff --git a/translations/zh.json b/translations/zh.json index 3c2711739..ffd2dd28c 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -297,5 +297,16 @@ "Edit newswire": "编辑新闻专线", "Add RSS feed links below.": "在下面添加RSS feed链接。", "Newswire RSS Feed": "Newswire RSS提要", - "Nicknames whose blog entries appear on the newswire.": "博客条目出现在新闻专线上的昵称。" + "Nicknames whose blog entries appear on the newswire.": "博客条目出现在新闻专线上的昵称。", + "Posts to be approved": "职位待批准", + "Discuss": "讨论", + "Moderator Discussion": "主持人讨论", + "Vote": "投票", + "Remove Vote": "删除投票", + "This is a news instance": "这是一个新闻实例", + "News": "新闻", + "Read more...": "阅读更多...", + "Edit News Post": "编辑新闻帖子", + "A list of editor nicknames. One per line.": "编辑者昵称列表。 每行一个。", + "Site Editors": "网站编辑" } diff --git a/utils.py b/utils.py index ce3f7012f..da689e60f 100644 --- a/utils.py +++ b/utils.py @@ -19,6 +19,60 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar +def createConfig(baseDir: str) -> None: + """Creates a configuration file + """ + configFilename = baseDir + '/config.json' + if os.path.isfile(configFilename): + return + configJson = { + } + saveJson(configJson, configFilename) + + +def setConfigParam(baseDir: str, variableName: str, variableValue) -> None: + """Sets a configuration value + """ + createConfig(baseDir) + configFilename = baseDir + '/config.json' + configJson = {} + if os.path.isfile(configFilename): + configJson = loadJson(configFilename) + configJson[variableName] = variableValue + saveJson(configJson, configFilename) + + +def getConfigParam(baseDir: str, variableName: str): + """Gets a configuration value + """ + createConfig(baseDir) + configFilename = baseDir + '/config.json' + configJson = loadJson(configFilename) + if configJson: + if configJson.get(variableName): + return configJson[variableName] + return None + + +def isSuspended(baseDir: str, nickname: str) -> bool: + """Returns true if the given nickname is suspended + """ + adminNickname = getConfigParam(baseDir, 'admin') + if not adminNickname: + return False + if nickname == adminNickname: + return False + + suspendedFilename = baseDir + '/accounts/suspended.txt' + if os.path.isfile(suspendedFilename): + with open(suspendedFilename, "r") as f: + lines = f.readlines() + for suspended in lines: + if suspended.strip('\n').strip('\r') == nickname: + return True + return False + + def getFollowersList(baseDir: str, nickname: str, domain: str, followFile='following.txt') -> []: @@ -168,10 +222,14 @@ def loadJsonOnionify(filename: str, domain: str, onionDomain: str, return jsonObject -def getStatusNumber() -> (str, str): +def getStatusNumber(publishedStr=None) -> (str, str): """Returns the status number and published date """ - currTime = datetime.datetime.utcnow() + if not publishedStr: + currTime = datetime.datetime.utcnow() + else: + currTime = \ + datetime.datetime.strptime(publishedStr, '%Y-%m-%dT%H:%M:%SZ') daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days # status is the number of seconds since epoch statusNumber = \ @@ -444,6 +502,70 @@ def followPerson(baseDir: str, nickname: str, domain: str, return True +def votesOnNewswireItem(status: []) -> int: + """Returns the number of votes on a newswire item + """ + totalVotes = 0 + for line in status: + if 'vote:' in line: + totalVotes += 1 + return totalVotes + + +def locateNewsVotes(baseDir: str, domain: str, + postUrl: str) -> str: + """Returns the votes filename for a news post + within the news user account + """ + postUrl = \ + postUrl.strip().replace('\n', '').replace('\r', '') + + # if this post in the shared inbox? + postUrl = removeIdEnding(postUrl.strip()).replace('/', '#') + + if postUrl.endswith('.json'): + postUrl = postUrl + '.votes' + else: + postUrl = postUrl + '.json.votes' + + accountDir = baseDir + '/accounts/news@' + domain + '/' + postFilename = accountDir + 'outbox/' + postUrl + if os.path.isfile(postFilename): + return postFilename + + return None + + +def locateNewsArrival(baseDir: str, domain: str, + postUrl: str) -> str: + """Returns the arrival time for a news post + within the news user account + """ + postUrl = \ + postUrl.strip().replace('\n', '').replace('\r', '') + + # if this post in the shared inbox? + postUrl = removeIdEnding(postUrl.strip()).replace('/', '#') + + if postUrl.endswith('.json'): + postUrl = postUrl + '.arrived' + else: + postUrl = postUrl + '.json.arrived' + + accountDir = baseDir + '/accounts/news@' + domain + '/' + postFilename = accountDir + 'outbox/' + postUrl + if os.path.isfile(postFilename): + with open(postFilename, 'r') as arrivalFile: + arrival = arrivalFile.read() + if arrival: + arrivalDate = \ + datetime.datetime.strptime(arrival, + "%Y-%m-%dT%H:%M:%SZ") + return arrivalDate + + return None + + def locatePost(baseDir: str, nickname: str, domain: str, postUrl: str, replies=False) -> str: """Returns the filename for the given status post url @@ -467,6 +589,12 @@ def locatePost(baseDir: str, nickname: str, domain: str, if os.path.isfile(postFilename): return postFilename + # check news posts + accountDir = baseDir + '/accounts/news' + '@' + domain + '/' + postFilename = accountDir + 'outbox/' + postUrl + if os.path.isfile(postFilename): + return postFilename + # is it in the announce cache? postFilename = baseDir + '/cache/announce/' + nickname + '/' + postUrl if os.path.isfile(postFilename): @@ -581,6 +709,16 @@ def deletePost(baseDir: str, httpPrefix: str, if os.path.isfile(muteFilename): os.remove(muteFilename) + # remove any votes file + votesFilename = postFilename + '.votes' + if os.path.isfile(votesFilename): + os.remove(votesFilename) + + # remove any arrived file + arrivedFilename = postFilename + '.arrived' + if os.path.isfile(arrivedFilename): + os.remove(arrivedFilename) + # remove cached html version of the post cachedPostFilename = \ getCachedPostFilename(baseDir, nickname, domain, postJsonObject) @@ -661,7 +799,7 @@ def deletePost(baseDir: str, httpPrefix: str, def validNickname(domain: str, nickname: str) -> bool: - forbiddenChars = ('.', ' ', '/', '?', ':', ';', '@') + forbiddenChars = ('.', ' ', '/', '?', ':', ';', '@', '#') for c in forbiddenChars: if c in nickname: return False @@ -671,7 +809,7 @@ def validNickname(domain: str, nickname: str) -> bool: 'public', 'followers', 'channel', 'calendar', 'tlreplies', 'tlmedia', 'tlblogs', - 'tlevents', + 'tlevents', 'tlblogs', 'moderation', 'activity', 'undo', 'reply', 'replies', 'question', 'like', 'likes', 'users', 'statuses', @@ -918,6 +1056,12 @@ def isBlogPost(postJsonObject: {}) -> bool: return True +def isNewsPost(postJsonObject: {}) -> bool: + """Is the given post a blog post? + """ + return postJsonObject.get('news') + + def searchBoxPosts(baseDir: str, nickname: str, domain: str, searchStr: str, maxResults: int, boxName='outbox') -> []: diff --git a/webinterface.py b/webinterface.py index 3afe1c2d1..849313626 100644 --- a/webinterface.py +++ b/webinterface.py @@ -30,6 +30,7 @@ from utils import getProtocolPrefixes from utils import searchBoxPosts from utils import isEventPost from utils import isBlogPost +from utils import isNewsPost from utils import updateRecentPostsCache from utils import getNicknameFromActor from utils import getDomainFromActor @@ -41,6 +42,8 @@ from utils import getDisplayName from utils import getCachedPostDirectory from utils import getCachedPostFilename from utils import loadJson +from utils import getConfigParam +from utils import votesOnNewswireItem from follow import isFollowingActor from webfinger import webfingerHandle from posts import isDM @@ -49,6 +52,7 @@ from posts import getUserUrl from posts import parseUserFeed from posts import populateRepliesJson from posts import isModerator +from posts import isEditor from posts import downloadAnnounce from session import getJson from auth import createPassword @@ -67,7 +71,6 @@ from content import addHtmlTags from content import replaceEmojiFromTags from content import removeLongWords from content import removeHtml -from config import getConfigParam from skills import getSkills from cache import getPersonFromCache from cache import storePersonInCache @@ -707,7 +710,8 @@ def htmlHashtagSearch(nickname: str, domain: str, port: int, postsPerPage: int, session, wfRequest: {}, personCache: {}, httpPrefix: str, projectVersion: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Show a page containing search results for a hashtag """ if hashtag.startswith('#'): @@ -774,7 +778,7 @@ def htmlHashtagSearch(nickname: str, domain: str, port: int, 'RSS 2.0' + iconsDir + '/logorss.png" />' if startIndex > 0: # previous page link @@ -827,6 +831,7 @@ def htmlHashtagSearch(nickname: str, domain: str, port: int, httpPrefix, projectVersion, 'search', YTReplacementDomain, + showPublishedDateOnly, showIndividualPostIcons, showIndividualPostIcons, False, False, False) @@ -1113,7 +1118,8 @@ def htmlHistorySearch(translate: {}, baseDir: str, wfRequest, personCache: {}, port: int, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Show a page containing search results for your post history """ if historysearch.startswith('!'): @@ -1185,6 +1191,7 @@ def htmlHistorySearch(translate: {}, baseDir: str, httpPrefix, projectVersion, 'search', YTReplacementDomain, + showPublishedDateOnly, showIndividualPostIcons, showIndividualPostIcons, False, False, False) @@ -1221,7 +1228,7 @@ def htmlEditLinks(translate: {}, baseDir: str, path: str, return '' # is the user a moderator? - if not isModerator(baseDir, nickname): + if not isEditor(baseDir, nickname): return '' cssFilename = baseDir + '/epicyon-links.css' @@ -1265,7 +1272,7 @@ def htmlEditLinks(translate: {}, baseDir: str, path: str, translate['One link per line. Description followed by the link.'] + \ '
' editLinksForm += \ - ' ' editLinksForm += \ '' @@ -1326,15 +1333,9 @@ def htmlEditNewswire(translate: {}, baseDir: str, path: str, with open(newswireFilename, 'r') as fp: newswireStr = fp.read() - # get the list of handles who are trusted to post to the newswire - newswireTrusted = '' - newswireTrustedFilename = baseDir + '/accounts/newswiretrusted.txt' - if os.path.isfile(newswireTrustedFilename): - with open(newswireTrustedFilename, "r") as trustFile: - newswireTrusted = trustFile.read() - editNewswireForm += \ '
' + editNewswireForm += \ ' ' + \ translate['Add RSS feed links below.'] + \ @@ -1343,14 +1344,6 @@ def htmlEditNewswire(translate: {}, baseDir: str, path: str, ' ' - editNewswireForm += \ - ' ' + \ - translate['Nicknames whose blog entries appear on the newswire.'] + \ - '
' - editNewswireForm += \ - ' ' - editNewswireForm += \ '
' @@ -1358,6 +1351,83 @@ def htmlEditNewswire(translate: {}, baseDir: str, path: str, return editNewswireForm +def htmlEditNewsPost(translate: {}, baseDir: str, path: str, + domain: str, port: int, + httpPrefix: str, postUrl: str) -> str: + """Edits a news post + """ + if '/users/' not in path: + return '' + pathOriginal = path + + nickname = getNicknameFromActor(path) + if not nickname: + return '' + + # is the user an editor? + if not isEditor(baseDir, nickname): + return '' + + postUrl = postUrl.replace('/', '#') + postFilename = locatePost(baseDir, nickname, domain, postUrl) + if not postFilename: + return '' + postJsonObject = loadJson(postFilename) + if not postJsonObject: + return '' + + cssFilename = baseDir + '/epicyon-links.css' + if os.path.isfile(baseDir + '/links.css'): + cssFilename = baseDir + '/links.css' + with open(cssFilename, 'r') as cssFile: + editCSS = cssFile.read() + if httpPrefix != 'https': + editCSS = \ + editCSS.replace('https://', httpPrefix + '://') + + editNewsPostForm = htmlHeader(cssFilename, editCSS) + editNewsPostForm += \ + '
\n' + editNewsPostForm += \ + '
\n' + editNewsPostForm += \ + '

' + translate['Edit News Post'] + '

' + editNewsPostForm += \ + '
\n' + editNewsPostForm += \ + ' ' + \ + '\n' + editNewsPostForm += \ + ' \n' + editNewsPostForm += \ + '
\n' + + editNewsPostForm += \ + '
' + + editNewsPostForm += \ + ' \n' + + newsPostTitle = postJsonObject['object']['summary'] + editNewsPostForm += \ + '
\n' + + newsPostContent = postJsonObject['object']['content'] + editNewsPostForm += \ + ' ' + + editNewsPostForm += \ + '
' + + editNewsPostForm += htmlFooter() + return editNewsPostForm + + def htmlEditProfile(translate: {}, baseDir: str, path: str, domain: str, port: int, httpPrefix: str) -> str: """Shows the edit profile screen @@ -1388,6 +1458,7 @@ def htmlEditProfile(translate: {}, baseDir: str, path: str, hideLikeButton = '' mediaInstanceStr = '' blogsInstanceStr = '' + newsInstanceStr = '' displayNickname = nickname bioStr = '' donateUrl = '' @@ -1446,12 +1517,21 @@ def htmlEditProfile(translate: {}, baseDir: str, path: str, if mediaInstance is True: mediaInstanceStr = 'checked' blogsInstanceStr = '' + newsInstanceStr = '' + + newsInstance = getConfigParam(baseDir, "newsInstance") + if newsInstance: + if newsInstance is True: + newsInstanceStr = 'checked' + blogsInstanceStr = '' + mediaInstanceStr = '' blogsInstance = getConfigParam(baseDir, "blogsInstance") if blogsInstance: if blogsInstance is True: blogsInstanceStr = 'checked' mediaInstanceStr = '' + newsInstanceStr = '' filterStr = '' filterFilename = \ @@ -1545,104 +1625,119 @@ def htmlEditProfile(translate: {}, baseDir: str, path: str, moderatorsStr = '' themesDropdown = '' adminNickname = getConfigParam(baseDir, 'admin') - if path.startswith('/users/' + adminNickname + '/'): - instanceDescription = \ - getConfigParam(baseDir, 'instanceDescription') - instanceDescriptionShort = \ - getConfigParam(baseDir, 'instanceDescriptionShort') - instanceTitle = \ - getConfigParam(baseDir, 'instanceTitle') - instanceStr = '
' - instanceStr += \ - ' ' - if instanceTitle: + if adminNickname: + if path.startswith('/users/' + adminNickname + '/'): + instanceDescription = \ + getConfigParam(baseDir, 'instanceDescription') + instanceDescriptionShort = \ + getConfigParam(baseDir, 'instanceDescriptionShort') + instanceTitle = \ + getConfigParam(baseDir, 'instanceTitle') + instanceStr = '
' instanceStr += \ - '
' - else: + ' ' + if instanceTitle: + instanceStr += \ + '
' + else: + instanceStr += \ + '
' instanceStr += \ - '
' - instanceStr += \ - ' ' - if instanceDescriptionShort: + ' ' + if instanceDescriptionShort: + instanceStr += \ + '
' + else: + instanceStr += \ + '
' instanceStr += \ - '
' - else: + ' ' + if instanceDescription: + instanceStr += \ + ' ' + else: + instanceStr += \ + ' ' instanceStr += \ - '
' - instanceStr += \ - ' ' - if instanceDescription: + ' ' instanceStr += \ - ' ' - else: - instanceStr += \ - ' ' - instanceStr += \ - ' ' - instanceStr += \ - ' ' - instanceStr += '
' + ' ' + instanceStr += '
' - moderators = '' - moderatorsFile = baseDir + '/accounts/moderators.txt' - if os.path.isfile(moderatorsFile): - with open(moderatorsFile, "r") as f: - moderators = f.read() - moderatorsStr = '
' - moderatorsStr += ' ' + translate['Moderators'] + '
' - moderatorsStr += ' ' + \ - translate['A list of moderator nicknames. One per line.'] - moderatorsStr += \ - ' ' - moderatorsStr += '
' + moderators = '' + moderatorsFile = baseDir + '/accounts/moderators.txt' + if os.path.isfile(moderatorsFile): + with open(moderatorsFile, "r") as f: + moderators = f.read() + moderatorsStr = '
' + moderatorsStr += ' ' + translate['Moderators'] + '
' + moderatorsStr += ' ' + \ + translate['A list of moderator nicknames. One per line.'] + moderatorsStr += \ + ' ' + moderatorsStr += '
' - themes = getThemesList() - themesDropdown = '
' - themesDropdown += ' ' + translate['Theme'] + '
' - grayscaleFilename = \ - baseDir + '/accounts/.grayscale' - grayscale = '' - if os.path.isfile(grayscaleFilename): - grayscale = 'checked' - themesDropdown += \ - ' ' + translate['Grayscale'] + '
' - themesDropdown += '
' - if os.path.isfile(baseDir + '/fonts/custom.woff') or \ - os.path.isfile(baseDir + '/fonts/custom.woff2') or \ - os.path.isfile(baseDir + '/fonts/custom.otf') or \ - os.path.isfile(baseDir + '/fonts/custom.ttf'): + editors = '' + editorsFile = baseDir + '/accounts/editors.txt' + if os.path.isfile(editorsFile): + with open(editorsFile, "r") as f: + editors = f.read() + editorsStr = '
' + editorsStr += ' ' + translate['Site Editors'] + '
' + editorsStr += ' ' + \ + translate['A list of editor nicknames. One per line.'] + editorsStr += \ + ' ' + editorsStr += '
' + + themes = getThemesList() + themesDropdown = '
' + themesDropdown += ' ' + translate['Theme'] + '
' + grayscaleFilename = \ + baseDir + '/accounts/.grayscale' + grayscale = '' + if os.path.isfile(grayscaleFilename): + grayscale = 'checked' themesDropdown += \ ' ' + \ - translate['Remove the custom font'] + '
' - themesDropdown += '
' - themeName = getConfigParam(baseDir, 'theme') - themesDropdown = \ - themesDropdown.replace('
' + themeName = getConfigParam(baseDir, 'theme') + themesDropdown = \ + themesDropdown.replace('
\n' editProfileForm += '
\n' @@ -1915,7 +2014,8 @@ def htmlEditProfile(translate: {}, baseDir: str, path: str, 'teams with an appropriate combination of skills.' editProfileForm += ' \n' - editProfileForm += skillsStr + themesDropdown + moderatorsStr + editProfileForm += skillsStr + themesDropdown + editProfileForm += moderatorsStr + editorsStr editProfileForm += '
\n' + instanceStr editProfileForm += '
\n' editProfileForm += '
\n' @@ -6039,6 +6395,7 @@ def htmlTimeline(defaultTimeline: str, httpPrefix, projectVersion, boxName, YTReplacementDomain, + showPublishedDateOnly, boxName != 'dm', showIndividualPostIcons, manuallyApproveFollowers, @@ -6063,7 +6420,9 @@ def htmlTimeline(defaultTimeline: str, # right column rightColumnStr = getRightColumnContent(baseDir, nickname, domainFull, httpPrefix, translate, iconsDir, - moderator, newswire) + moderator, editor, + newswire, positiveVoting, + False, None) tlStr += ' ' + \ rightColumnStr + ' \n' tlStr += ' \n' @@ -6105,7 +6464,8 @@ def htmlShares(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the shares timeline as html """ manuallyApproveFollowers = \ @@ -6117,7 +6477,10 @@ def htmlShares(defaultTimeline: str, nickname, domain, port, None, 'tlshares', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - False, YTReplacementDomain, newswire) + False, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlInbox(defaultTimeline: str, @@ -6128,7 +6491,8 @@ def htmlInbox(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the inbox as html """ manuallyApproveFollowers = \ @@ -6140,7 +6504,10 @@ def htmlInbox(defaultTimeline: str, nickname, domain, port, inboxJson, 'inbox', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlBookmarks(defaultTimeline: str, @@ -6151,7 +6518,8 @@ def htmlBookmarks(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the bookmarks as html """ manuallyApproveFollowers = \ @@ -6163,7 +6531,10 @@ def htmlBookmarks(defaultTimeline: str, nickname, domain, port, bookmarksJson, 'tlbookmarks', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlEvents(defaultTimeline: str, @@ -6174,7 +6545,8 @@ def htmlEvents(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the events as html """ manuallyApproveFollowers = \ @@ -6186,7 +6558,10 @@ def htmlEvents(defaultTimeline: str, nickname, domain, port, bookmarksJson, 'tlevents', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlInboxDMs(defaultTimeline: str, @@ -6197,7 +6572,8 @@ def htmlInboxDMs(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the DM timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6205,7 +6581,8 @@ def htmlInboxDMs(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'dm', allowDeletion, httpPrefix, projectVersion, False, minimal, - YTReplacementDomain, newswire) + YTReplacementDomain, showPublishedDateOnly, + newswire, False, False, positiveVoting) def htmlInboxReplies(defaultTimeline: str, @@ -6216,7 +6593,8 @@ def htmlInboxReplies(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the replies timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6224,7 +6602,10 @@ def htmlInboxReplies(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlreplies', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlInboxMedia(defaultTimeline: str, @@ -6235,7 +6616,8 @@ def htmlInboxMedia(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the media timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6243,7 +6625,10 @@ def htmlInboxMedia(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlmedia', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) def htmlInboxBlogs(defaultTimeline: str, @@ -6254,7 +6639,8 @@ def htmlInboxBlogs(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the blogs timeline as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6262,7 +6648,34 @@ def htmlInboxBlogs(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'tlblogs', allowDeletion, httpPrefix, projectVersion, False, - minimal, YTReplacementDomain, newswire) + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, False, False, + positiveVoting) + + +def htmlInboxNews(defaultTimeline: str, + recentPostsCache: {}, maxRecentPosts: int, + translate: {}, pageNumber: int, itemsPerPage: int, + session, baseDir: str, wfRequest: {}, personCache: {}, + nickname: str, domain: str, port: int, inboxJson: {}, + allowDeletion: bool, + httpPrefix: str, projectVersion: str, + minimal: bool, YTReplacementDomain: str, + showPublishedDateOnly: bool, + newswire: {}, moderator: bool, editor: bool, + positiveVoting: bool) -> str: + """Show the news timeline as html + """ + return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, + translate, pageNumber, + itemsPerPage, session, baseDir, wfRequest, personCache, + nickname, domain, port, inboxJson, 'tlnews', + allowDeletion, httpPrefix, projectVersion, False, + minimal, YTReplacementDomain, + showPublishedDateOnly, + newswire, moderator, editor, + positiveVoting) def htmlModeration(defaultTimeline: str, @@ -6273,7 +6686,8 @@ def htmlModeration(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the moderation feed as html """ return htmlTimeline(defaultTimeline, recentPostsCache, maxRecentPosts, @@ -6281,7 +6695,8 @@ def htmlModeration(defaultTimeline: str, itemsPerPage, session, baseDir, wfRequest, personCache, nickname, domain, port, inboxJson, 'moderation', allowDeletion, httpPrefix, projectVersion, True, False, - YTReplacementDomain, newswire) + YTReplacementDomain, showPublishedDateOnly, + newswire, False, False, positiveVoting) def htmlOutbox(defaultTimeline: str, @@ -6292,7 +6707,8 @@ def htmlOutbox(defaultTimeline: str, allowDeletion: bool, httpPrefix: str, projectVersion: str, minimal: bool, YTReplacementDomain: str, - newswire: {}) -> str: + showPublishedDateOnly: bool, + newswire: {}, positiveVoting: bool) -> str: """Show the Outbox as html """ manuallyApproveFollowers = \ @@ -6303,7 +6719,8 @@ def htmlOutbox(defaultTimeline: str, nickname, domain, port, outboxJson, 'outbox', allowDeletion, httpPrefix, projectVersion, manuallyApproveFollowers, minimal, - YTReplacementDomain, newswire) + YTReplacementDomain, showPublishedDateOnly, + newswire, False, False, positiveVoting) def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int, @@ -6312,7 +6729,8 @@ def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int, nickname: str, domain: str, port: int, authorized: bool, postJsonObject: {}, httpPrefix: str, projectVersion: str, likedBy: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Show an individual post as html """ iconsDir = getIconsDir(baseDir) @@ -6357,6 +6775,7 @@ def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int, None, True, False, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, + showPublishedDateOnly, False, authorized, False, False, False) messageId = removeIdEnding(postJsonObject['id']) @@ -6381,6 +6800,7 @@ def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int, None, True, False, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, + showPublishedDateOnly, False, authorized, False, False, False) + postStr @@ -6408,6 +6828,7 @@ def htmlIndividualPost(recentPostsCache: {}, maxRecentPosts: int, None, True, False, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, + showPublishedDateOnly, False, authorized, False, False, False) cssFilename = baseDir + '/epicyon-profile.css' @@ -6426,7 +6847,8 @@ def htmlPostReplies(recentPostsCache: {}, maxRecentPosts: int, session, wfRequest: {}, personCache: {}, nickname: str, domain: str, port: int, repliesJson: {}, httpPrefix: str, projectVersion: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Show the replies to an individual post as html """ iconsDir = getIconsDir(baseDir) @@ -6442,6 +6864,7 @@ def htmlPostReplies(recentPostsCache: {}, maxRecentPosts: int, None, True, False, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, + showPublishedDateOnly, False, False, False, False, False) cssFilename = baseDir + '/epicyon-profile.css' @@ -6530,7 +6953,8 @@ def htmlDeletePost(recentPostsCache: {}, maxRecentPosts: int, httpPrefix: str, projectVersion: str, wfRequest: {}, personCache: {}, callingDomain: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Shows a screen asking to confirm the deletion of a post """ if '/statuses/' not in messageId: @@ -6575,6 +6999,7 @@ def htmlDeletePost(recentPostsCache: {}, maxRecentPosts: int, None, True, False, httpPrefix, projectVersion, 'outbox', YTReplacementDomain, + showPublishedDateOnly, False, False, False, False, False) deletePostStr += '
' deletePostStr += \ @@ -7549,7 +7974,8 @@ def htmlProfileAfterSearch(recentPostsCache: {}, maxRecentPosts: int, profileHandle: str, session, cachedWebfingers: {}, personCache: {}, debug: bool, projectVersion: str, - YTReplacementDomain: str) -> str: + YTReplacementDomain: str, + showPublishedDateOnly: bool) -> str: """Show a profile page after a search for a fediverse address """ if '/users/' in profileHandle or \ @@ -7757,6 +8183,7 @@ def htmlProfileAfterSearch(recentPostsCache: {}, maxRecentPosts: int, item, avatarUrl, False, False, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, + showPublishedDateOnly, False, False, False, False, False) i += 1 if i >= 20: diff --git a/website/EN/index.html b/website/EN/index.html index b0f39e741..c7517e5ac 100644 --- a/website/EN/index.html +++ b/website/EN/index.html @@ -1268,7 +1268,7 @@

You will need python version 3.7 or later.

On a Debian based system:

-

sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-pyld python3-django-timezone-field python3-pyqrcode python3-png python3-bandit libimage-exiftool-perl certbot nginx

+

sudo apt install -y tor python3-socks imagemagick python3-numpy python3-setuptools python3-crypto python3-pycryptodome python3-dateutil python3-pil.imagetk python3-idna python3-requests python3-flake8 python3-django-timezone-field python3-pyqrcode python3-png python3-bandit libimage-exiftool-perl certbot nginx