diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..25aacffde --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +build/ +dist/ +*.egg-info/ diff --git a/content.py b/content.py index fef7d66e9..4f20267e8 100644 --- a/content.py +++ b/content.py @@ -14,6 +14,8 @@ from utils import getImageExtensions from utils import loadJson from utils import fileLastModified from utils import getLinkPrefixes +from utils import dangerousMarkup +from petnames import getPetName def removeHtmlTag(htmlStr: str, tag: str) -> str: @@ -153,38 +155,6 @@ def htmlReplaceQuoteMarks(content: str) -> str: return newContent -def dangerousMarkup(content: str, allowLocalNetworkAccess: bool) -> bool: - """Returns true if the given content contains dangerous html markup - """ - if '<' not in content: - return False - if '>' not in content: - return False - contentSections = content.split('<') - invalidPartials = () - if not allowLocalNetworkAccess: - invalidPartials = ('localhost', '127.0.', '192.168', '10.0.') - invalidStrings = ('script', 'canvas', 'style', 'abbr', - 'frame', 'iframe', 'html', 'body', - 'hr', 'allow-popups', 'allow-scripts') - for markup in contentSections: - if '>' not in markup: - continue - markup = markup.split('>')[0].strip() - for partialMatch in invalidPartials: - if partialMatch in markup: - return True - if ' ' not in markup: - for badStr in invalidStrings: - if badStr in markup: - return True - else: - for badStr in invalidStrings: - if badStr + ' ' in markup: - return True - return False - - def dangerousCSS(filename: str, allowLocalNetworkAccess: bool) -> bool: """Returns true is the css file contains code which can create security problems @@ -489,7 +459,7 @@ def tagExists(tagType: str, tagName: str, tags: {}) -> bool: return False -def _addMention(wordStr: str, httpPrefix: str, following: str, +def _addMention(wordStr: str, httpPrefix: str, following: str, petnames: str, replaceMentions: {}, recipients: [], tags: {}) -> bool: """Detects mentions and adds them to the replacements dict and recipients list @@ -501,9 +471,12 @@ def _addMention(wordStr: str, httpPrefix: str, following: str, # if no domain was specified. eg. @nick possibleNickname = possibleHandle for follow in following: - if follow.startswith(possibleNickname + '@'): - replaceDomain = \ - follow.replace('\n', '').replace('\r', '').split('@')[1] + if '@' not in follow: + continue + followNick = follow.split('@')[0] + if possibleNickname == followNick: + followStr = follow.replace('\n', '').replace('\r', '') + replaceDomain = followStr.split('@')[1] recipientActor = httpPrefix + "://" + \ replaceDomain + "/users/" + possibleNickname if recipientActor not in recipients: @@ -519,6 +492,34 @@ def _addMention(wordStr: str, httpPrefix: str, following: str, "\" class=\"u-url mention\">@" + possibleNickname + \ "" return True + # try replacing petnames with mentions + followCtr = 0 + for follow in following: + if '@' not in follow: + followCtr += 1 + continue + pet = petnames[followCtr].replace('\n', '') + if pet: + if possibleNickname == pet: + followStr = follow.replace('\n', '').replace('\r', '') + replaceNickname = followStr.split('@')[0] + replaceDomain = followStr.split('@')[1] + recipientActor = httpPrefix + "://" + \ + replaceDomain + "/users/" + replaceNickname + if recipientActor not in recipients: + recipients.append(recipientActor) + tags[wordStr] = { + 'href': recipientActor, + 'name': wordStr, + 'type': 'Mention' + } + replaceMentions[wordStr] = \ + "@" + \ + replaceNickname + "" + return True + followCtr += 1 return False possibleNickname = None possibleDomain = None @@ -752,10 +753,14 @@ def addHtmlTags(baseDir: str, httpPrefix: str, # read the following list so that we can detect just @nick # in addition to @nick@domain following = None + petnames = None if '@' in words: if os.path.isfile(followingFilename): with open(followingFilename, "r") as f: following = f.readlines() + for handle in following: + pet = getPetName(baseDir, nickname, domain, handle) + petnames.append(pet + '\n') # extract mentions and tags from words longWordsList = [] @@ -769,7 +774,7 @@ def addHtmlTags(baseDir: str, httpPrefix: str, longWordsList.append(wordStr) firstChar = wordStr[0] if firstChar == '@': - if _addMention(wordStr, httpPrefix, following, + if _addMention(wordStr, httpPrefix, following, petnames, replaceMentions, recipients, hashtags): prevWordStr = '' continue diff --git a/daemon.py b/daemon.py index 5ad42d349..0c70a77fb 100644 --- a/daemon.py +++ b/daemon.py @@ -217,10 +217,10 @@ from utils import urlPermitted from utils import loadJson from utils import saveJson from utils import isSuspended +from utils import dangerousMarkup from manualapprove import manualDenyFollowRequest from manualapprove import manualApproveFollowRequest from announce import createAnnounce -from content import dangerousMarkup from content import replaceEmojiFromTags from content import addHtmlTags from content import extractMediaInFormPOST @@ -1136,7 +1136,7 @@ class PubServer(BaseHTTPRequestHandler): """ if self.server.restartInboxQueueInProgress: self._503() - print('Message arrrived but currently restarting inbox queue') + print('Message arrived but currently restarting inbox queue') self.server.POSTbusy = False return 2 @@ -2614,7 +2614,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self.server.YTReplacementDomain, self.server.showPublishedDateOnly, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -2666,7 +2667,8 @@ class PubServer(BaseHTTPRequestHandler): port, self.server.YTReplacementDomain, self.server.showPublishedDateOnly, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) if historyStr: msg = historyStr.encode('utf-8') msglen = len(msg) @@ -2733,6 +2735,8 @@ class PubServer(BaseHTTPRequestHandler): return else: showPublishedDateOnly = self.server.showPublishedDateOnly + allowLocalNetworkAccess = \ + self.server.allowLocalNetworkAccess profileStr = \ htmlProfileAfterSearch(self.server.cssCache, self.server.recentPostsCache, @@ -2753,7 +2757,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.YTReplacementDomain, showPublishedDateOnly, self.server.defaultTimeline, - self.server.peertubeInstances) + self.server.peertubeInstances, + allowLocalNetworkAccess) if profileStr: msg = profileStr.encode('utf-8') msglen = len(msg) @@ -5674,7 +5679,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self.server.YTReplacementDomain, self.server.showPublishedDateOnly, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -6636,7 +6642,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.personCache, callingDomain, self.server.YTReplacementDomain, self.server.showPublishedDateOnly, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) if deleteStr: deleteStrLen = len(deleteStr) self._set_headers('text/html', deleteStrLen, @@ -6840,7 +6847,8 @@ class PubServer(BaseHTTPRequestHandler): projectVersion, ytDomain, self.server.showPublishedDateOnly, - peertubeInstances) + peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -6926,7 +6934,8 @@ class PubServer(BaseHTTPRequestHandler): projectVersion, ytDomain, self.server.showPublishedDateOnly, - peertubeInstances) + peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7013,6 +7022,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + self.server.allowLocalNetworkAccess, actorJson['roles'], None, None) msg = msg.encode('utf-8') @@ -7077,6 +7087,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.showPublishedDateOnly iconsAsButtons = \ self.server.iconsAsButtons + allowLocalNetworkAccess = \ + self.server.allowLocalNetworkAccess msg = \ htmlProfile(self.server.rssIconAtTop, self.server.cssCache, @@ -7097,6 +7109,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + allowLocalNetworkAccess, actorJson['skills'], None, None) msg = msg.encode('utf-8') @@ -7208,6 +7221,8 @@ class PubServer(BaseHTTPRequestHandler): peertubeInstances = \ self.server.peertubeInstances cssCache = self.server.cssCache + allowLocalNetworkAccess = \ + self.server.allowLocalNetworkAccess msg = \ htmlIndividualPost(cssCache, recentPostsCache, @@ -7227,7 +7242,8 @@ class PubServer(BaseHTTPRequestHandler): likedBy, ytDomain, showPublishedDateOnly, - peertubeInstances) + peertubeInstances, + allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7329,6 +7345,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.showPublishedDateOnly peertubeInstances = \ self.server.peertubeInstances + allowLocalNetworkAccess = \ + self.server.allowLocalNetworkAccess msg = \ htmlIndividualPost(self.server.cssCache, recentPostsCache, @@ -7348,7 +7366,8 @@ class PubServer(BaseHTTPRequestHandler): likedBy, ytDomain, showPublishedDateOnly, - peertubeInstances) + peertubeInstances, + allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7481,7 +7500,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, 'show status done', @@ -7608,7 +7628,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7728,7 +7749,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7849,7 +7871,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7970,7 +7993,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8100,7 +8124,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8226,7 +8251,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8313,7 +8339,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8417,7 +8444,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8541,7 +8569,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8657,7 +8686,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8763,7 +8793,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, moderationActionStr, self.server.themeName, - self.server.peertubeInstances) + self.server.peertubeInstances, + self.server.allowLocalNetworkAccess) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8863,6 +8894,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + self.server.allowLocalNetworkAccess, shares, pageNumber, sharesPerPage) msg = msg.encode('utf-8') @@ -8959,6 +8991,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + self.server.allowLocalNetworkAccess, following, pageNumber, followsPerPage).encode('utf-8') @@ -9055,6 +9088,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + self.server.allowLocalNetworkAccess, followers, pageNumber, followsPerPage).encode('utf-8') @@ -9174,6 +9208,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, self.server.peertubeInstances, + self.server.allowLocalNetworkAccess, None, None).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, diff --git a/deploy/onion b/deploy/onion index cf277b4e6..ea99ffea4 100755 --- a/deploy/onion +++ b/deploy/onion @@ -16,10 +16,10 @@ if [[ "$1" == 'remove' ]]; then rm "/etc/nginx/sites-availale/${username}" rm -rf ${install_destination} if [ -d /var/www/cache ]; then - rm -rf /var/www/cache + rm -rf /var/www/cache fi if [ -d /srv/http/cache ]; then - rm -rf /srv/http/cache + rm -rf /srv/http/cache fi userdel -r ${username} echo 'Epicyon onion instance removed' @@ -37,18 +37,36 @@ if [ -f /usr/bin/pacman ]; then pacman -Syy pacman -S --noconfirm tor python-pip python-pysocks python-pycryptodome \ imagemagick python-pillow python-requests \ - perl-image-exiftool python-numpy python-dateutil \ - certbot flake8 git qrencode bandit + perl-image-exiftool python-numpy python-dateutil \ + certbot flake8 git qrencode bandit pip3 install pyLD pyqrcode pypng else apt-get update apt-get -y install imagemagick python3-crypto python3-pycryptodome \ - python3-dateutil python3-idna python3-requests \ - python3-numpy python3-pil.imagetk python3-pip \ - python3-setuptools python3-socks python3-idna \ - libimage-exiftool-perl python3-flake8 python3-pyld \ - python3-django-timezone-field tor nginx git qrencode \ - python3-pyqrcode python3-png python3-bandit + python3-dateutil python3-idna python3-requests \ + python3-numpy python3-pil.imagetk python3-pip \ + python3-setuptools python3-socks python3-idna \ + libimage-exiftool-perl python3-flake8 python3-pyld \ + python3-django-timezone-field tor nginx git qrencode \ + python3-pyqrcode python3-png python3-bandit +fi + +if [[ "$(uname -a)" == *'Debian'* ]]; then + echo 'Fixing the tor daemon' + { echo '[Unit]'; + echo 'Description=Anonymizing overlay network for TCP (multi-instance-master)'; + echo ''; + echo '[Service]'; + echo 'Type=simple'; + echo 'User=root'; + echo 'Group=debian-tor'; + echo 'ExecStart=/usr/bin/tor --defaults-torrc /usr/share/tor/tor-service-defaults-torrc -f /etc/tor/torrc --RunAsDaemon 0'; + echo ''; + echo '[Install]'; + echo 'WantedBy=multi-user.target'; } > /lib/systemd/system/tor.service + cp /lib/systemd/system/tor.service /root/tor.service + systemctl daemon-reload + systemctl restart tor fi echo 'Cloning the epicyon repo' @@ -56,8 +74,8 @@ if [ ! -d ${install_destination} ]; then git clone https://gitlab.com/bashrc2/epicyon ${install_destination} if [ ! -d ${install_destination} ]; then - echo 'Epicyon repo failed to clone' - exit 3 + echo 'Epicyon repo failed to clone' + exit 3 fi fi @@ -79,6 +97,7 @@ if [ ! -d /etc/torrc.d ]; then fi if ! grep -q '%include /etc/torrc.d' /etc/tor/torrc; then echo '%include /etc/torrc.d' >> /etc/tor/torrc + systemctl restart tor fi if [ ! -f /etc/torrc.d/epicyon ]; then @@ -185,7 +204,7 @@ if [ ! -f /etc/nginx/nginx.conf ]; then echo '}'; } > /etc/nginx/nginx.conf else if ! grep -q 'include /etc/nginx/sites-enabled' /etc/nginx/nginx.conf; then - echo 'include /etc/nginx/sites-enabled/*.conf;' >> /etc/nginx/nginx.conf + echo 'include /etc/nginx/sites-enabled/*.conf;' >> /etc/nginx/nginx.conf fi fi if [ ! -d /etc/nginx/conf.d ]; then @@ -200,25 +219,25 @@ fi if [ -f /usr/bin/pacman ]; then if [ ! -f /lib/systemd/system/nginx.service ]; then - echo 'Creating nginx daemon' - { echo '[Unit]'; - echo 'Description=A high performance web server and a reverse proxy server'; - echo 'Documentation=man:nginx(8)'; - echo 'After=network.target nss-lookup.target'; - echo '' - echo '[Service]'; - echo 'Type=forking'; - echo 'PIDFile=/run/nginx.pid'; - echo "ExecStartPre=$(which nginx) -t -q -g 'daemon on; master_process on;'"; - echo "ExecStart=$(which nginx) -g 'daemon on; master_process on;'"; - echo "ExecReload=$(which nginx) -g 'daemon on; master_process on;' -s reload"; - echo 'ExecStop=-/sbin/start-stop-daemon --quiet --stop --retry QUIT/5 --pidfile /run/nginx.pid'; - echo 'TimeoutStopSec=5'; - echo 'KillMode=mixed'; - echo ''; - echo '[Install]'; - echo 'WantedBy=multi-user.target'; } > /etc/systemd/system/nginx.service - systemctl enable nginx + echo 'Creating nginx daemon' + { echo '[Unit]'; + echo 'Description=A high performance web server and a reverse proxy server'; + echo 'Documentation=man:nginx(8)'; + echo 'After=network.target nss-lookup.target'; + echo '' + echo '[Service]'; + echo 'Type=forking'; + echo 'PIDFile=/run/nginx.pid'; + echo "ExecStartPre=$(which nginx) -t -q -g 'daemon on; master_process on;'"; + echo "ExecStart=$(which nginx) -g 'daemon on; master_process on;'"; + echo "ExecReload=$(which nginx) -g 'daemon on; master_process on;' -s reload"; + echo 'ExecStop=-/sbin/start-stop-daemon --quiet --stop --retry QUIT/5 --pidfile /run/nginx.pid'; + echo 'TimeoutStopSec=5'; + echo 'KillMode=mixed'; + echo ''; + echo '[Install]'; + echo 'WantedBy=multi-user.target'; } > /etc/systemd/system/nginx.service + systemctl enable nginx fi fi @@ -257,7 +276,7 @@ echo "Creating nginx virtual host for ${ONION_DOMAIN}" echo ' index index.html;'; echo ''; echo ' location /newsmirror {'; - echo ' root /var/www/${ONION_DOMAIN}/htdocs;'; + echo " root /var/www/${ONION_DOMAIN}/htdocs;"; echo ' try_files $uri =404;'; echo ' }'; echo ''; diff --git a/epicyon-profile.css b/epicyon-profile.css index 7e1fd85e4..f97f70677 100644 --- a/epicyon-profile.css +++ b/epicyon-profile.css @@ -51,8 +51,8 @@ --font-size-tox: 16px; --font-size-tox2: 18px; --time-color: #aaa; - --time-vertical-align: 4px; - --time-vertical-align-mobile: 25px; + --time-vertical-align: 0%; + --time-vertical-align-mobile: 1.5%; --publish-button-text: #FFFFFF; --button-margin: 5px; --button-left-margin: none; @@ -96,6 +96,7 @@ --column-right-width: 10vw; --column-left-mobile-margin: 2%; --column-left-top-margin: 0; + --column-right-top-margin: 0; --column-left-header-style: uppercase; --column-left-header-background: #555; --column-left-header-color: #fff; @@ -1156,7 +1157,7 @@ div.container { .col-right img.rightColImg { background: var(--column-left-color); width: 100%; - margin: 0 0; + margin-top: var(--column-right-top-margin); padding: 0 0; } .likesCount { diff --git a/epicyon.py b/epicyon.py index e5f3ecbf4..b776e0bbd 100644 --- a/epicyon.py +++ b/epicyon.py @@ -868,7 +868,11 @@ configPort = getConfigParam(baseDir, 'port') if configPort: port = configPort else: - port = 8085 + if domain.endswith('.onion') or \ + domain.endswith('.i2p'): + port = 80 + else: + port = 443 configProxyPort = getConfigParam(baseDir, 'proxyPort') if configProxyPort: @@ -1613,6 +1617,10 @@ if args.addaccount: if os.path.isdir(baseDir + '/deactivated/' + nickname + '@' + domain): print('Account is deactivated') sys.exit() + if domain.endswith('.onion') or \ + domain.endswith('.i2p'): + port = 80 + httpPrefix = 'http' createPerson(baseDir, nickname, domain, port, httpPrefix, True, not args.noapproval, args.password.strip()) if os.path.isdir(baseDir + '/accounts/' + nickname + '@' + domain): diff --git a/inbox.py b/inbox.py index b572a3043..c213e0d22 100644 --- a/inbox.py +++ b/inbox.py @@ -54,6 +54,7 @@ from blocking import isBlockedDomain from filters import isFiltered from utils import updateAnnounceCollection from utils import undoAnnounceCollectionEntry +from utils import dangerousMarkup from httpsig import messageContentDigest from posts import validContentWarning from posts import downloadAnnounce @@ -69,7 +70,6 @@ from media import replaceYouTube from git import isGitPatch from git import receiveGitPatch from followingCalendar import receivingCalendarEvents -from content import dangerousMarkup from happening import saveEventPost from delete import removeOldHashtags from follow import isFollowingActor @@ -151,7 +151,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, postJsonObject: {}, allowDeletion: bool, boxname: str, showPublishedDateOnly: bool, - peertubeInstances: []) -> None: + peertubeInstances: [], + allowLocalNetworkAccess: bool) -> None: """Converts the json post into html and stores it in a cache This enables the post to be quickly displayed later """ @@ -168,7 +169,7 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, avatarUrl, True, allowDeletion, httpPrefix, __version__, boxname, None, showPublishedDateOnly, - peertubeInstances, + peertubeInstances, allowLocalNetworkAccess, not isDM(postJsonObject), True, True, False, True) @@ -1259,7 +1260,8 @@ def _receiveAnnounce(recentPostsCache: {}, sendThreads: [], postLog: [], cachedWebfingers: {}, personCache: {}, messageJson: {}, federationList: [], debug: bool, translate: {}, - YTReplacementDomain: str) -> bool: + YTReplacementDomain: str, + allowLocalNetworkAccess: bool) -> bool: """Receives an announce activity within the POST section of HTTPServer """ if messageJson['type'] != 'Announce': @@ -1338,7 +1340,8 @@ def _receiveAnnounce(recentPostsCache: {}, postJsonObject = downloadAnnounce(session, baseDir, httpPrefix, nickname, domain, messageJson, __version__, translate, - YTReplacementDomain) + YTReplacementDomain, + allowLocalNetworkAccess) if not postJsonObject: if domain not in messageJson['object'] and \ onionDomain not in messageJson['object']: @@ -2119,7 +2122,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, messageJson, federationList, debug, translate, - YTReplacementDomain): + YTReplacementDomain, + allowLocalNetworkAccess): if debug: print('DEBUG: Announce accepted from ' + actor) @@ -2299,7 +2303,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, if isImageMedia(session, baseDir, httpPrefix, nickname, domain, postJsonObject, - translate, YTReplacementDomain): + translate, YTReplacementDomain, + allowLocalNetworkAccess): # media index will be updated updateIndexList.append('tlmedia') if isBlogPost(postJsonObject): @@ -2349,7 +2354,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, allowDeletion, boxname, showPublishedDateOnly, - peertubeInstances) + peertubeInstances, + allowLocalNetworkAccess) if debug: timeDiff = \ str(int((time.time() - htmlCacheStartTime) * diff --git a/newsdaemon.py b/newsdaemon.py index 0a6a8f5cc..84f2f4bc4 100644 --- a/newsdaemon.py +++ b/newsdaemon.py @@ -23,7 +23,6 @@ from newswire import getDictFromNewswire # from posts import sendSignedJson from posts import createNewsPost from posts import archivePostsForPerson -from content import dangerousMarkup from content import validHashTag from utils import removeHtml from utils import getFullDomain @@ -31,6 +30,7 @@ from utils import loadJson from utils import saveJson from utils import getStatusNumber from utils import clearFromPostCaches +from utils import dangerousMarkup from inbox import storeHashTags from session import createSession diff --git a/outbox.py b/outbox.py index ce0bf15d9..e50000386 100644 --- a/outbox.py +++ b/outbox.py @@ -17,6 +17,7 @@ from posts import sendToNamedAddresses from utils import getFullDomain from utils import removeIdEnding from utils import getDomainFromActor +from utils import dangerousMarkup from blocking import isBlockedDomain from blocking import outboxBlock from blocking import outboxUndoBlock @@ -36,7 +37,6 @@ from bookmarks import outboxUndoBookmark from delete import outboxDelete from shares import outboxShareUpload from shares import outboxUndoShareUpload -from content import dangerousMarkup def postMessageToOutbox(messageJson: {}, postToNickname: str, diff --git a/posts.py b/posts.py index 6dd6c7e31..a0920284c 100644 --- a/posts.py +++ b/posts.py @@ -55,6 +55,7 @@ from utils import locateNewsVotes from utils import locateNewsArrival from utils import votesOnNewswireItem from utils import removeHtml +from utils import dangerousMarkup from media import attachMedia from media import replaceYouTube from content import tagExists @@ -291,7 +292,13 @@ def getPersonBox(baseDir: str, session, wfRequest: {}, avatarUrl = personJson['icon']['url'] displayName = None if personJson.get('name'): - displayName = removeHtml(personJson['name']) + displayName = personJson['name'] + if dangerousMarkup(personJson['name'], False): + displayName = '*ADVERSARY*' + elif isFiltered(baseDir, + nickname, domain, + displayName): + displayName = '*FILTERED*' # have they moved? if personJson.get('movedTo'): displayName += ' ⌂' @@ -1824,11 +1831,16 @@ def threadSendPost(session, postJsonStr: str, federationList: [], for attempt in range(20): postResult = None unauthorized = False + if debug: + print('Getting postJsonString for ' + inboxUrl) try: postResult, unauthorized = \ postJsonString(session, postJsonStr, federationList, inboxUrl, signatureHeaderJson, debug) + if debug: + print('Obtained postJsonString for ' + inboxUrl + + ' unauthorized: ' + str(unauthorized)) except Exception as e: print('ERROR: postJsonString failed ' + str(e)) if unauthorized: @@ -2908,7 +2920,8 @@ def isDM(postJsonObject: {}) -> bool: def isImageMedia(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, postJsonObject: {}, translate: {}, - YTReplacementDomain: str) -> bool: + YTReplacementDomain: str, + allowLocalNetworkAccess: bool) -> bool: """Returns true if the given post has attached image media """ if postJsonObject['type'] == 'Announce': @@ -2916,7 +2929,8 @@ def isImageMedia(session, baseDir: str, httpPrefix: str, downloadAnnounce(session, baseDir, httpPrefix, nickname, domain, postJsonObject, __version__, translate, - YTReplacementDomain) + YTReplacementDomain, + allowLocalNetworkAccess) if postJsonAnnounce: postJsonObject = postJsonAnnounce if postJsonObject['type'] != 'Create': @@ -3831,7 +3845,8 @@ def _rejectAnnounce(announceFilename: str): def downloadAnnounce(session, baseDir: str, httpPrefix: str, nickname: str, domain: str, postJsonObject: {}, projectVersion: str, - translate: {}, YTReplacementDomain: str) -> {}: + translate: {}, YTReplacementDomain: str, + allowLocalNetworkAccess: bool) -> {}: """Download the post referenced by an announce """ if not postJsonObject.get('object'): @@ -3911,20 +3926,16 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, if '/statuses/' not in announcedJson['id']: _rejectAnnounce(announceFilename) return None - if '/users/' not in announcedJson['id'] and \ - '/accounts/' not in announcedJson['id'] and \ - '/channel/' not in announcedJson['id'] and \ - '/profile/' not in announcedJson['id']: + if not hasUsersPath(announcedJson['id']): _rejectAnnounce(announceFilename) return None if not announcedJson.get('type'): _rejectAnnounce(announceFilename) - # pprint(announcedJson) return None if announcedJson['type'] != 'Note' and \ announcedJson['type'] != 'Article': + # You can only announce Note or Article types _rejectAnnounce(announceFilename) - # pprint(announcedJson) return None if not announcedJson.get('content'): _rejectAnnounce(announceFilename) @@ -3935,16 +3946,25 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, if not validPostDate(announcedJson['published']): _rejectAnnounce(announceFilename) return None - if isFiltered(baseDir, nickname, domain, announcedJson['content']): + + # Check the content of the announce + contentStr = announcedJson['content'] + if dangerousMarkup(contentStr, allowLocalNetworkAccess): _rejectAnnounce(announceFilename) return None + + if isFiltered(baseDir, nickname, domain, contentStr): + _rejectAnnounce(announceFilename) + return None + # remove any long words - announcedJson['content'] = \ - removeLongWords(announcedJson['content'], 40, []) + contentStr = removeLongWords(contentStr, 40, []) # remove text formatting, such as bold/italics - announcedJson['content'] = \ - removeTextFormatting(announcedJson['content']) + contentStr = removeTextFormatting(contentStr) + + # set the content after santitization + announcedJson['content'] = contentStr # wrap in create to be consistent with other posts announcedJson = \ @@ -3952,8 +3972,8 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, actorNickname, actorDomain, actorPort, announcedJson) if announcedJson['type'] != 'Create': + # Create wrap failed _rejectAnnounce(announceFilename) - # pprint(announcedJson) return None # labelAccusatoryPost(postJsonObject, translate) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..9787c3bdf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..4c55942dc --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[metadata] +name = epicyon +version = 1.2.0 + +[options] +packages = . +install_requires = + crypto + idna<3,>=2.5 + numpy + pillow + pycryptodome + pyqrcode + python-dateutil + requests + socks diff --git a/tests.py b/tests.py index f2ef8b22d..83ecabca7 100644 --- a/tests.py +++ b/tests.py @@ -49,6 +49,7 @@ from utils import saveJson from utils import getStatusNumber from utils import getFollowersOfPerson from utils import removeHtml +from utils import dangerousMarkup from follow import followerOfPerson from follow import unfollowAccount from follow import unfollowerOfAccount @@ -77,7 +78,6 @@ from inbox import validInboxFilenames from categories import guessHashtagCategory from content import htmlReplaceEmailQuote from content import htmlReplaceQuoteMarks -from content import dangerousMarkup from content import dangerousCSS from content import addWebLinks from content import replaceEmojiFromTags @@ -95,6 +95,7 @@ from newswire import getNewswireTags from newswire import parseFeedDate from mastoapiv1 import getMastoApiV1IdFromNickname from mastoapiv1 import getNicknameFromMastoApiV1Id +from webapp_post import prepareHtmlPostNickname testServerAliceRunning = False testServerBobRunning = False @@ -3072,9 +3073,25 @@ def testDomainHandling(): assert decodedHost(testDomain) == "españa.icom.museum" +def testPrepareHtmlPostNickname(): + print('testPrepareHtmlPostNickname') + postHtml = ' bool: + """Returns true if the given content contains dangerous html markup + """ + if '<' not in content: + return False + if '>' not in content: + return False + contentSections = content.split('<') + invalidPartials = () + if not allowLocalNetworkAccess: + invalidPartials = ('localhost', '127.0.', '192.168', '10.0.') + invalidStrings = ('script', 'canvas', 'style', 'abbr', + 'frame', 'iframe', 'html', 'body', + 'hr', 'allow-popups', 'allow-scripts') + for markup in contentSections: + if '>' not in markup: + continue + markup = markup.split('>')[0].strip() + for partialMatch in invalidPartials: + if partialMatch in markup: + return True + if ' ' not in markup: + for badStr in invalidStrings: + if badStr in markup: + return True + else: + for badStr in invalidStrings: + if badStr + ' ' in markup: + return True + return False + + def getDisplayName(baseDir: str, actor: str, personCache: {}) -> str: """Returns the display name for the given actor """ @@ -561,9 +593,10 @@ def getDisplayName(baseDir: str, actor: str, personCache: {}) -> str: actor = actor.split('/statuses/')[0] if not personCache.get(actor): return None + nameFound = None if personCache[actor].get('actor'): if personCache[actor]['actor'].get('name'): - return personCache[actor]['actor']['name'] + nameFound = personCache[actor]['actor']['name'] else: # Try to obtain from the cached actors cachedActorFilename = \ @@ -572,8 +605,11 @@ def getDisplayName(baseDir: str, actor: str, personCache: {}) -> str: actorJson = loadJson(cachedActorFilename, 1) if actorJson: if actorJson.get('name'): - return(actorJson['name']) - return None + nameFound = actorJson['name'] + if nameFound: + if dangerousMarkup(nameFound, False): + nameFound = "*ADVERSARY*" + return nameFound def getNicknameFromActor(actor: str) -> str: @@ -1721,6 +1757,11 @@ def siteIsActive(url: str) -> bool: """ if not url.startswith('http'): return False + if '.onion/' in url or '.i2p/' in url or \ + url.endswith('.onion') or \ + url.endswith('.i2p'): + # skip this check for onion and i2p + return True try: req = urllib.request.Request(url) urllib.request.urlopen(req, timeout=10) # nosec diff --git a/webapp_column_left.py b/webapp_column_left.py index ee7a6de77..14651564f 100644 --- a/webapp_column_left.py +++ b/webapp_column_left.py @@ -90,7 +90,7 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, htmlStr += \ '\n
\n' + \ ' \n' + \ '
\n' @@ -115,7 +115,7 @@ def getLeftColumnContent(baseDir: str, nickname: str, domainFull: str, '/users/' + nickname + '/editlinks">' + \ '' + \ - translate['Edit Links'] + '\n' @@ -268,6 +268,7 @@ def htmlLinksMobile(cssCache: {}, baseDir: str, htmlStr += \ '' + \ '\n' htmlStr += '
\n' @@ -333,7 +334,8 @@ def htmlEditLinks(cssCache: {}, translate: {}, baseDir: str, path: str, '\n' - editLinksForm += '\n' + \ '\n' diff --git a/webapp_column_right.py b/webapp_column_right.py index 2a5b778d0..b245a5c6e 100644 --- a/webapp_column_right.py +++ b/webapp_column_right.py @@ -92,7 +92,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, htmlStr += \ '\n
\n' + \ ' \n' + \ '
\n' @@ -123,7 +123,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, '/users/' + nickname + '/editnewswire">' + \ '' + \ - translate['Edit newswire'] + '\n' else: @@ -133,7 +133,7 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, '/users/' + nickname + '/editnewswire">' + \ '' + \ - translate['Edit newswire'] + '\n' @@ -142,14 +142,14 @@ def getRightColumnContent(baseDir: str, nickname: str, domainFull: str, ' ' + \ '' + \ - translate['Hashtag Categories RSS Feed'] + '\n' rssIconStr += \ ' ' + \ '' + \ - translate['Newswire RSS Feed'] + '\n' if rssIconAtTop: @@ -241,6 +241,7 @@ def _htmlNewswire(baseDir: str, newswire: {}, nickname: str, moderator: bool, if faviconUrl: faviconLink = \ '' moderatedItem = item[5] htmlStr += separatorStr @@ -265,6 +266,7 @@ def _htmlNewswire(baseDir: str, newswire: {}, nickname: str, moderator: bool, '/newswireunvote=' + dateStrLink + '" ' + \ 'title="' + translate['Remove Vote'] + '">' htmlStr += '

\n' else: htmlStr += ' ' @@ -290,8 +292,9 @@ def _htmlNewswire(baseDir: str, newswire: {}, nickname: str, moderator: bool, htmlStr += '' - htmlStr += '' + htmlStr += '' htmlStr += '

\n' else: htmlStr += '

' + \ @@ -354,7 +357,8 @@ def htmlCitations(baseDir: str, nickname: str, domain: str, '\n' - htmlStr += '\n' htmlStr += \ @@ -464,6 +468,7 @@ def htmlNewswireMobile(cssCache: {}, baseDir: str, nickname: str, htmlStr += \ '' + \ '\n' htmlStr += '

\n' @@ -531,8 +536,8 @@ def htmlEditNewswire(cssCache: {}, translate: {}, baseDir: str, path: str, translate['Switch to timeline view'] + '" alt="' + \ translate['Switch to timeline view'] + '">\n' editNewswireForm += '\n' + \ - '' + '/users/' + nickname + '/' + bannerFile + '" ' + \ + 'alt="" />\n' editNewswireForm += \ '
str: + peertubeInstances: [], + allowLocalNetworkAccess: bool) -> str: """Shows a screen asking to confirm the deletion of a post """ if '/statuses/' not in messageId: @@ -70,7 +71,7 @@ def htmlConfirmDelete(cssCache: {}, httpPrefix, projectVersion, 'outbox', YTReplacementDomain, showPublishedDateOnly, - peertubeInstances, + peertubeInstances, allowLocalNetworkAccess, False, False, False, False, False) deletePostStr += '
' deletePostStr += \ diff --git a/webapp_create_post.py b/webapp_create_post.py index d15bc8880..0c8ca1fd1 100644 --- a/webapp_create_post.py +++ b/webapp_create_post.py @@ -572,7 +572,7 @@ def htmlNewPost(cssCache: {}, mediaInstance: bool, translate: {}, translate['Switch to timeline view'] + '" alt="' + \ translate['Switch to timeline view'] + '">\n' newPostForm += '\n' + \ + '/users/' + nickname + '/' + bannerFile + '" alt="" />\n' + \ '\n' mentionsStr = '' diff --git a/webapp_frontscreen.py b/webapp_frontscreen.py index 473e223a1..97aadb938 100644 --- a/webapp_frontscreen.py +++ b/webapp_frontscreen.py @@ -29,7 +29,8 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, projectVersion: str, YTReplacementDomain: str, showPublishedDateOnly: bool, - peertubeInstances: []) -> str: + peertubeInstances: [], + allowLocalNetworkAccess: bool) -> str: """Shows posts on the front screen of a news instance These should only be public blog posts from the features timeline which is the blog timeline of the news actor @@ -69,6 +70,7 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, YTReplacementDomain, showPublishedDateOnly, peertubeInstances, + allowLocalNetworkAccess, False, False, False, True, False) if postStr: profileStr += postStr + separatorStr @@ -91,6 +93,7 @@ def htmlFrontScreen(rssIconAtTop: bool, showPublishedDateOnly: bool, newswire: {}, theme: str, peertubeInstances: [], + allowLocalNetworkAccess: bool, extraJson=None, pageNumber=None, maxItemsPerPage=None) -> str: """Show the news instance front screen @@ -155,7 +158,8 @@ def htmlFrontScreen(rssIconAtTop: bool, projectVersion, YTReplacementDomain, showPublishedDateOnly, - peertubeInstances) + licenseStr + peertubeInstances, + allowLocalNetworkAccess) + licenseStr # Footer which is only used for system accounts profileFooterStr = ' \n' diff --git a/webapp_hashtagswarm.py b/webapp_hashtagswarm.py index 076e54108..ebb9aa2cc 100644 --- a/webapp_hashtagswarm.py +++ b/webapp_hashtagswarm.py @@ -263,7 +263,7 @@ def htmlSearchHashtagCategory(cssCache: {}, translate: {}, if os.path.isfile(searchBannerFilename): htmlStr += '\n' htmlStr += '\n' + actor + '/' + searchBannerFile + '" alt="" />\n' htmlStr += '\n' return separatorStr diff --git a/webfinger.py b/webfinger.py index 45d247449..451aaaae4 100644 --- a/webfinger.py +++ b/webfinger.py @@ -221,6 +221,7 @@ def webfingerLookup(path: str, baseDir: str, handle = None if 'resource=acct:' in path: handle = path.split('resource=acct:')[1].strip() + handle = urllib.parse.unquote(handle) if debug: print('DEBUG: WEBFINGER handle ' + handle) else: