diff --git a/announce.py b/announce.py index 8c8486a98..dccfb20ea 100644 --- a/announce.py +++ b/announce.py @@ -32,6 +32,8 @@ def outboxAnnounce(recentPostsCache: {}, """ if not messageJson.get('actor'): return False + if not isinstance(messageJson['actor'], str): + return False if not messageJson.get('type'): return False if not messageJson.get('object'): @@ -39,6 +41,8 @@ def outboxAnnounce(recentPostsCache: {}, if messageJson['type'] == 'Announce': if not isinstance(messageJson['object'], str): return False + if isSelfAnnounce(messageJson): + return False nickname = getNicknameFromActor(messageJson['actor']) if not nickname: print('WARN: no nickname found in ' + messageJson['actor']) @@ -386,3 +390,21 @@ def outboxUndoAnnounce(recentPostsCache: {}, messageJson['actor'], domain, debug) if debug: print('DEBUG: post undo announce via c2s - ' + postFilename) + + +def isSelfAnnounce(postJsonObject: {}) -> bool: + """Is the given post a self announce? + """ + if not postJsonObject.get('actor'): + return False + if not postJsonObject.get('type'): + return False + if postJsonObject['type'] != 'Announce': + return False + if not postJsonObject.get('object'): + return False + if not isinstance(postJsonObject['actor'], str): + return False + if not isinstance(postJsonObject['object'], str): + return False + return postJsonObject['actor'] in postJsonObject['object'] diff --git a/daemon.py b/daemon.py index fb6a3f4f5..7ad7d5ee4 100644 --- a/daemon.py +++ b/daemon.py @@ -606,9 +606,10 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('WWW-Authenticate', 'title="Login to Epicyon", Basic realm="epicyon"') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _logout_headers(self, fileFormat: str, length: int, @@ -620,9 +621,10 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('WWW-Authenticate', 'title="Login to Epicyon", Basic realm="epicyon"') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _logout_redirect(self, redirect: str, cookie: str, @@ -637,9 +639,10 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) self.send_header('Content-Length', '0') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _set_headers_base(self, fileFormat: str, length: int, cookie: str, @@ -657,16 +660,17 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Cookie', cookieStr) self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') self.send_header('X-Clacks-Overhead', 'GNU Natalie Nguyen') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.send_header('Accept-Ranges', 'none') def _set_headers(self, fileFormat: str, length: int, cookie: str, callingDomain: str) -> None: self._set_headers_base(fileFormat, length, cookie, callingDomain) - self.send_header('Cache-Control', 'public, max-age=0') + # self.send_header('Cache-Control', 'public, max-age=0') self.end_headers() def _set_headers_head(self, fileFormat: str, length: int, etag: str, @@ -680,7 +684,7 @@ class PubServer(BaseHTTPRequestHandler): data, cookie: str, callingDomain: str) -> None: datalen = len(data) self._set_headers_base(fileFormat, datalen, cookie, callingDomain) - self.send_header('Cache-Control', 'public, max-age=86400') + # self.send_header('Cache-Control', 'public, max-age=86400') etag = None if os.path.isfile(mediaFilename + '.etag'): try: @@ -745,9 +749,10 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Host', callingDomain) self.send_header('InstanceID', self.server.instanceId) self.send_header('Content-Length', '0') - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() def _httpReturnCode(self, httpCode: int, httpDescription: str, @@ -767,9 +772,10 @@ class PubServer(BaseHTTPRequestHandler): self.send_header('Content-Type', 'text/html; charset=utf-8') msgLenStr = str(len(msg)) self.send_header('Content-Length', msgLenStr) - self.send_header('X-Robots-Tag', - 'noindex, nofollow, noarchive, nosnippet') - self.send_header('Referrer-Policy', 'origin') + # self.send_header('X-Robots-Tag', + # 'noindex, nofollow, noarchive, nosnippet') + # self.send_header('Cache-Control', 'public') + # self.send_header('Referrer-Policy', 'origin') self.end_headers() if not self._write(msg): print('Error when showing ' + str(httpCode)) diff --git a/desktop_client.py b/desktop_client.py index de1ad92b0..caa39516d 100644 --- a/desktop_client.py +++ b/desktop_client.py @@ -836,7 +836,7 @@ def _desktopShowProfile(session, nickname: str, domain: str, isHttp = False if 'http://' in actor: isHttp = True - actorJson = getActorJson(actor, isHttp, False, False, True) + actorJson, asHeader = getActorJson(actor, isHttp, False, False, True) _desktopShowActor(baseDir, actorJson, translate, systemLanguage, screenreader, espeak) @@ -854,7 +854,7 @@ def _desktopShowProfileFromHandle(session, nickname: str, domain: str, """Shows the profile for a handle Returns the actor json """ - actorJson = getActorJson(handle, False, False, False, True) + actorJson, asHeader = getActorJson(handle, False, False, False, True) _desktopShowActor(baseDir, actorJson, translate, systemLanguage, screenreader, espeak) diff --git a/inbox.py b/inbox.py index 38978dd4e..05ba550e9 100644 --- a/inbox.py +++ b/inbox.py @@ -80,6 +80,7 @@ from delete import removeOldHashtags from categories import guessHashtagCategory from context import hasValidContext from speaker import updateSpeaker +from announce import isSelfAnnounce def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None: @@ -1359,6 +1360,10 @@ def _receiveAnnounce(recentPostsCache: {}, '"users" or "profile" missing from actor in ' + messageJson['type']) return False + if isSelfAnnounce(messageJson): + if debug: + print('DEBUG: self-boost rejected') + return False if not hasUsersPath(messageJson['object']): if debug: print('DEBUG: ' + diff --git a/metadata.py b/metadata.py index e073692c3..1c117a672 100644 --- a/metadata.py +++ b/metadata.py @@ -60,6 +60,25 @@ def metaDataNodeInfo(baseDir: str, return nodeinfo +def _getStatusCount(baseDir: str) -> int: + """Get the total number of posts + """ + statusCtr = 0 + accountsDir = baseDir + '/accounts' + for subdir, dirs, files in os.walk(accountsDir): + for acct in dirs: + if '@' not in acct: + continue + if 'inbox@' in acct or 'news@' in acct: + continue + acctDir = os.path.join(accountsDir, acct + '/outbox') + for subdir2, dirs2, files2 in os.walk(acctDir): + statusCtr += len(files2) + break + break + return statusCtr + + def metaDataInstance(instanceTitle: str, instanceDescriptionShort: str, instanceDescription: str, @@ -94,30 +113,21 @@ def metaDataInstance(instanceTitle: str, 'avatar': adminActor['icon']['url'], 'avatar_static': adminActor['icon']['url'], 'bot': isBot, - 'created_at': '2019-07-01T10:30:00Z', 'display_name': adminActor['name'], - 'emojis': [], - 'fields': [], - 'followers_count': 1, - 'following_count': 1, 'header': adminActor['image']['url'], 'header_static': adminActor['image']['url'], - 'id': '1', - 'last_status_at': '2019-07-01T10:30:00Z', 'locked': adminActor['manuallyApprovesFollowers'], 'note': '

Admin of ' + domain + '

', - 'statuses_count': 1, 'url': url, 'username': adminActor['preferredUsername'] }, 'description': instanceDescription, - 'email': 'admin@' + domain, 'languages': [systemLanguage], 'registrations': registration, 'short_description': instanceDescriptionShort, 'stats': { 'domain_count': 2, - 'status_count': 1, + 'status_count': _getStatusCount(baseDir), 'user_count': noOfAccounts(baseDir) }, 'thumbnail': httpPrefix + '://' + domainFull + '/login.png', diff --git a/person.py b/person.py index 84d0af727..9dea15d95 100644 --- a/person.py +++ b/person.py @@ -1205,7 +1205,7 @@ def setPersonNotes(baseDir: str, nickname: str, domain: str, def getActorJson(handle: str, http: bool, gnunet: bool, - debug: bool, quiet=False) -> {}: + debug: bool, quiet=False) -> ({}, {}): """Returns the actor json """ if debug: @@ -1216,15 +1216,16 @@ def getActorJson(handle: str, http: bool, gnunet: bool, handle.startswith('http') or \ handle.startswith('dat'): # format: https://domain/@nick + originalHandle = handle + if not hasUsersPath(originalHandle): + if not quiet or debug: + print('getActorJson: Expected actor format: ' + + 'https://domain/@nick or https://domain/users/nick') + return None, None prefixes = getProtocolPrefixes() for prefix in prefixes: handle = handle.replace(prefix, '') handle = handle.replace('/@', '/users/') - if not hasUsersPath(handle): - if not quiet or debug: - print('getActorJson: Expected actor format: ' + - 'https://domain/@nick or https://domain/users/nick') - return None if '/users/' in handle: nickname = handle.split('/users/')[1] nickname = nickname.replace('\n', '').replace('\r', '') @@ -1245,18 +1246,27 @@ def getActorJson(handle: str, http: bool, gnunet: bool, nickname = handle.split('/u/')[1] nickname = nickname.replace('\n', '').replace('\r', '') domain = handle.split('/u/')[0] + elif '://' in originalHandle: + domain = originalHandle.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in originalHandle: + return None, None + nickname = originalHandle.split('://' + domain + '/')[1] + if '/' in nickname or '.' in nickname: + return None, None else: # format: @nick@domain if '@' not in handle: if not quiet: print('getActorJson Syntax: --actor nickname@domain') - return None + return None, None if handle.startswith('@'): handle = handle[1:] if '@' not in handle: if not quiet: print('getActorJsonSyntax: --actor nickname@domain') - return None + return None, None nickname = handle.split('@')[0] domain = handle.split('@')[1] domain = domain.replace('\n', '').replace('\r', '') @@ -1287,12 +1297,12 @@ def getActorJson(handle: str, http: bool, gnunet: bool, if not wfRequest: if not quiet: print('getActorJson Unable to webfinger ' + handle) - return None + return None, None if not isinstance(wfRequest, dict): if not quiet: print('getActorJson Webfinger for ' + handle + ' did not return a dict. ' + str(wfRequest)) - return None + return None, None if not quiet: pprint(wfRequest) @@ -1306,12 +1316,12 @@ def getActorJson(handle: str, http: bool, gnunet: bool, else: if debug: print('No users path in ' + handle) - return None + return None, None profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/activity+json; profile="' + profileStr + '"' - } + headersList = ( + "activity+json", "ld+json", "jrd+json" + ) if not personUrl: personUrl = getUserUrl(wfRequest, 0, debug) if nickname == domain: @@ -1322,36 +1332,26 @@ def getActorJson(handle: str, http: bool, gnunet: bool, personUrl = personUrl.replace('/u/', '/actor/') if not personUrl: # try single user instance - personUrl = httpPrefix + '://' + domain - profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } + personUrl = httpPrefix + '://' + domain + '/' + nickname + headersList = ( + "ld+json", "jrd+json", "activity+json" + ) if '/channel/' in personUrl or '/accounts/' in personUrl: - profileStr = 'https://www.w3.org/ns/activitystreams' + headersList = ( + "ld+json", "jrd+json", "activity+json" + ) + if debug: + print('personUrl: ' + personUrl) + for headerType in headersList: + headerMimeType = 'application/' + headerType asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } - - personJson = \ - getJson(session, personUrl, asHeader, None, - debug, __version__, httpPrefix, None, 20, quiet) - if personJson: - if not quiet: - pprint(personJson) - return personJson - else: - profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/jrd+json; profile="' + profileStr + '"' + 'Accept': headerMimeType + '; profile="' + profileStr + '"' } personJson = \ getJson(session, personUrl, asHeader, None, - debug, __version__, httpPrefix, None) - if not quiet: - if personJson: - print('getActorJson returned actor') + debug, __version__, httpPrefix, None, 20, quiet) + if personJson: + if not quiet: pprint(personJson) - else: - print('Failed to get ' + personUrl) - return personJson + return personJson, asHeader + return None, None diff --git a/pgp.py b/pgp.py index 81720f3a3..722adb707 100644 --- a/pgp.py +++ b/pgp.py @@ -336,7 +336,7 @@ def _getPGPPublicKeyFromActor(handle: str, actorJson=None) -> str: public key specified """ if not actorJson: - actorJson = getActorJson(handle, False, False, False, True) + actorJson, asHeader = getActorJson(handle, False, False, False, True) if not actorJson: return None if not actorJson.get('attachment'): @@ -476,7 +476,7 @@ def pgpPublicKeyUpload(baseDir: str, session, if debug: print('Getting actor for ' + handle) - actorJson = getActorJson(handle, False, False, debug, True) + actorJson, asHeader = getActorJson(handle, False, False, debug, True) if not actorJson: if debug: print('No actor returned for ' + handle) diff --git a/posts.py b/posts.py index 1d4801d78..726357b59 100644 --- a/posts.py +++ b/posts.py @@ -3924,6 +3924,9 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, return None if not isinstance(postJsonObject['object'], str): return None + # ignore self-boosts + if postJsonObject['actor'] in postJsonObject['object']: + return None # get the announced post announceCacheDir = baseDir + '/cache/announce/' + nickname diff --git a/utils.py b/utils.py index 148199f68..428a6c5f3 100644 --- a/utils.py +++ b/utils.py @@ -15,7 +15,6 @@ import json import idna import locale from pprint import pprint -from calendar import monthrange from followingCalendar import addPersonToCalendar from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes @@ -98,6 +97,16 @@ def hasUsersPath(pathStr: str) -> bool: for usersStr in usersList: if '/' + usersStr + '/' in pathStr: return True + if '://' in pathStr: + domain = pathStr.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in pathStr: + return False + nickname = pathStr.split('://' + domain + '/')[1] + if '/' in nickname or '.' in nickname: + return False + return True return False @@ -842,6 +851,16 @@ def getNicknameFromActor(actor: str) -> str: elif '@' in actor: nickStr = actor.split('@')[0] return nickStr + elif '://' in actor: + domain = actor.split('://')[1] + if '/' in domain: + domain = domain.split('/')[0] + if '://' + domain + '/' not in actor: + return None + nickStr = actor.split('://' + domain + '/')[1] + if '/' in nickStr or '.' in nickStr: + return None + return nickStr return None nickStr = actor.split('/users/')[1].replace('@', '') if '/' not in nickStr: diff --git a/webapp_profile.py b/webapp_profile.py index b97d0e9a9..00ff979dc 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -10,7 +10,6 @@ import os from pprint import pprint from utils import getOccupationName from utils import getLockedAccount -from utils import hasUsersPath from utils import getFullDomain from utils import isArtist from utils import isDormant @@ -24,10 +23,9 @@ from utils import getImageFormats from skills import getSkills from theme import getThemesList from person import personBoxJson +from person import getActorJson from webfinger import webfingerHandle -from session import getJson from posts import parseUserFeed -from posts import getUserUrl from posts import getPersonBox from donate import getDonationUrl from xmpp import getXmppAddress @@ -74,46 +72,20 @@ def htmlProfileAfterSearch(cssCache: {}, accessKeys: {}) -> str: """Show a profile page after a search for a fediverse address """ - if hasUsersPath(profileHandle) or '/@' in profileHandle: - searchNickname = getNicknameFromActor(profileHandle) - searchDomain, searchPort = getDomainFromActor(profileHandle) - else: - if '@' not in profileHandle: - if debug: - print('DEBUG: no @ in ' + profileHandle) - return None - if profileHandle.startswith('@'): - profileHandle = profileHandle[1:] - if '@' not in profileHandle: - if debug: - print('DEBUG: no @ in ' + profileHandle) - return None - searchNickname = profileHandle.split('@')[0] - searchDomain = profileHandle.split('@')[1] - searchPort = None - if ':' in searchDomain: - searchPortStr = searchDomain.split(':')[1] - if searchPortStr.isdigit(): - searchPort = int(searchPortStr) - searchDomain = searchDomain.split(':')[0] - if searchPort: - if debug: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain) + ':' + - str(searchPort)) - else: - if debug: - print('DEBUG: Search for handle ' + - str(searchNickname) + '@' + str(searchDomain)) - if not searchNickname: - if debug: - print('DEBUG: No nickname found in ' + profileHandle) - return None - if not searchDomain: - if debug: - print('DEBUG: No domain found in ' + profileHandle) + http = False + gnunet = False + if httpPrefix == 'http': + http = True + elif httpPrefix == 'gnunet': + gnunet = True + profileJson, asHeader = \ + getActorJson(profileHandle, http, gnunet, debug, False) + if not profileJson: return None + personUrl = profileJson['id'] + searchDomain, searchPort = getDomainFromActor(personUrl) + searchNickname = getNicknameFromActor(personUrl) searchDomainFull = getFullDomain(searchDomain, searchPort) profileStr = '' @@ -121,57 +93,6 @@ def htmlProfileAfterSearch(cssCache: {}, if os.path.isfile(baseDir + '/epicyon.css'): cssFilename = baseDir + '/epicyon.css' - wf = \ - webfingerHandle(session, - searchNickname + '@' + searchDomainFull, - httpPrefix, cachedWebfingers, - domain, projectVersion, debug) - if not wf: - if debug: - print('DEBUG: Unable to webfinger ' + - searchNickname + '@' + searchDomainFull) - print('DEBUG: cachedWebfingers ' + str(cachedWebfingers)) - print('DEBUG: httpPrefix ' + httpPrefix) - print('DEBUG: domain ' + domain) - return None - if not isinstance(wf, dict): - if debug: - print('WARN: Webfinger search for ' + - searchNickname + '@' + searchDomainFull + - ' did not return a dict. ' + - str(wf)) - return None - - personUrl = None - if wf.get('errors'): - personUrl = httpPrefix + '://' + \ - searchDomainFull + '/users/' + searchNickname - - profileStr = 'https://www.w3.org/ns/activitystreams' - asHeader = { - 'Accept': 'application/activity+json; profile="' + profileStr + '"' - } - if not personUrl: - personUrl = getUserUrl(wf, 0, debug) - if not personUrl: - # try single user instance - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } - personUrl = httpPrefix + '://' + searchDomainFull - profileJson = \ - getJson(session, personUrl, asHeader, None, debug, - projectVersion, httpPrefix, domain) - if not profileJson: - asHeader = { - 'Accept': 'application/ld+json; profile="' + profileStr + '"' - } - profileJson = \ - getJson(session, personUrl, asHeader, None, debug, - projectVersion, httpPrefix, domain) - if not profileJson: - print('DEBUG: No actor returned from ' + personUrl) - return None avatarUrl = '' if profileJson.get('icon'): if profileJson['icon'].get('url'): diff --git a/webapp_timeline.py b/webapp_timeline.py index 9500ffb6e..50ec00fbc 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -31,6 +31,7 @@ from webapp_column_left import getLeftColumnContent from webapp_column_right import getRightColumnContent from webapp_headerbuttons import headerButtonsTimeline from posts import isModerator +from announce import isSelfAnnounce def _logTimelineTiming(enableTimingLog: bool, timelineStartTime, @@ -684,6 +685,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, # is the actor who sent this post snoozed? if isPersonSnoozed(baseDir, nickname, domain, item['actor']): continue + if isSelfAnnounce(item): + continue # is the post in the memory cache of recent ones? currTlStr = None