From 838d853a59b0c1d53624f1f1486ea986c2a83218 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 10:57:44 +0000 Subject: [PATCH 01/31] Tidying of users path detection --- acceptreject.py | 6 ++--- announce.py | 6 ++--- auth.py | 6 ++--- blocking.py | 11 +++------- bookmarks.py | 11 +++------- daemon.py | 8 +++---- delete.py | 6 ++--- epicyon.py | 11 +++------- follow.py | 21 +++++------------- inbox.py | 56 ++++++++++------------------------------------- like.py | 6 ++--- posts.py | 6 ++--- utils.py | 10 +++++++++ webapp_profile.py | 7 ++---- 14 files changed, 53 insertions(+), 118 deletions(-) diff --git a/acceptreject.py b/acceptreject.py index 6aa4fc3d0..8db41534c 100644 --- a/acceptreject.py +++ b/acceptreject.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from utils import hasUsersPath from utils import getFullDomain from utils import urlPermitted from utils import getDomainFromActor @@ -182,10 +183,7 @@ def receiveAcceptReject(session, baseDir: str, if debug: print('DEBUG: ' + messageJson['type'] + ' has no actor') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor in ' + messageJson['type'] + '. Assuming single user instance.') diff --git a/announce.py b/announce.py index 5364ba483..08b1bdd20 100644 --- a/announce.py +++ b/announce.py @@ -6,6 +6,7 @@ __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" +from utils import hasUsersPath from utils import getFullDomain from utils import getStatusNumber from utils import createOutboxDir @@ -143,10 +144,7 @@ def createAnnounce(session, baseDir: str, federationList: [], announceNickname = None announceDomain = None announcePort = None - if '/users/' in objectUrl or \ - '/accounts/' in objectUrl or \ - '/channel/' in objectUrl or \ - '/profile/' in objectUrl: + if hasUsersPath(objectUrl): announceNickname = getNicknameFromActor(objectUrl) announceDomain, announcePort = getDomainFromActor(objectUrl) diff --git a/auth.py b/auth.py index 841141279..e0690a983 100644 --- a/auth.py +++ b/auth.py @@ -12,6 +12,7 @@ import binascii import os import secrets from utils import isSystemAccount +from utils import hasUsersPath def _hashPassword(password: str) -> str: @@ -89,10 +90,7 @@ def authorizeBasic(baseDir: str, path: str, authHeader: str, print('DEBUG: basic auth - Authorixation header does not ' + 'contain a space character') return False - if '/users/' not in path and \ - '/accounts/' not in path and \ - '/channel/' not in path and \ - '/profile/' not in path: + if not hasUsersPath(path): if debug: print('DEBUG: basic auth - ' + 'path for Authorization does not contain a user') diff --git a/blocking.py b/blocking.py index c34641fc2..6fa7af3dd 100644 --- a/blocking.py +++ b/blocking.py @@ -7,6 +7,7 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import os +from utils import hasUsersPath from utils import getFullDomain from utils import removeIdEnding from utils import isEvil @@ -246,10 +247,7 @@ def outboxBlock(baseDir: str, httpPrefix: str, if debug: print('DEBUG: c2s block object is not a status') return - if '/users/' not in messageId and \ - '/accounts/' not in messageId and \ - '/channel/' not in messageId and \ - '/profile/' not in messageId: + if not hasUsersPath(messageId): if debug: print('DEBUG: c2s block object has no nickname') return @@ -321,10 +319,7 @@ def outboxUndoBlock(baseDir: str, httpPrefix: str, if debug: print('DEBUG: c2s undo block object is not a status') return - if '/users/' not in messageId and \ - '/accounts/' not in messageId and \ - '/channel/' not in messageId and \ - '/profile/' not in messageId: + if not hasUsersPath(messageId): if debug: print('DEBUG: c2s undo block object has no nickname') return diff --git a/bookmarks.py b/bookmarks.py index a9bf0c457..d6f9817c0 100644 --- a/bookmarks.py +++ b/bookmarks.py @@ -8,6 +8,7 @@ __status__ = "Production" import os from pprint import pprint +from utils import hasUsersPath from utils import getFullDomain from utils import removeIdEnding from utils import removePostFromCache @@ -255,10 +256,7 @@ def bookmark(recentPostsCache: {}, bookmarkedPostNickname = getNicknameFromActor(acBm) bookmarkedPostDomain, bookmarkedPostPort = getDomainFromActor(acBm) else: - if '/users/' in objectUrl or \ - '/accounts/' in objectUrl or \ - '/channel/' in objectUrl or \ - '/profile/' in objectUrl: + if hasUsersPath(objectUrl): ou = objectUrl bookmarkedPostNickname = getNicknameFromActor(ou) bookmarkedPostDomain, bookmarkedPostPort = getDomainFromActor(ou) @@ -322,10 +320,7 @@ def undoBookmark(recentPostsCache: {}, bookmarkedPostNickname = getNicknameFromActor(acBm) bookmarkedPostDomain, bookmarkedPostPort = getDomainFromActor(acBm) else: - if '/users/' in objectUrl or \ - '/accounts/' in objectUrl or \ - '/channel/' in objectUrl or \ - '/profile/' in objectUrl: + if hasUsersPath(objectUrl): ou = objectUrl bookmarkedPostNickname = getNicknameFromActor(ou) bookmarkedPostDomain, bookmarkedPostPort = getDomainFromActor(ou) diff --git a/daemon.py b/daemon.py index 8537ea5b5..15bdd70db 100644 --- a/daemon.py +++ b/daemon.py @@ -171,9 +171,10 @@ from shares import getSharesFeedForPerson from shares import addShare from shares import removeShare from shares import expireShares +from categories import setHashtagCategory +from utils import hasUsersPath from utils import getFullDomain from utils import removeHtml -from categories import setHashtagCategory from utils import isEditor from utils import getImageExtensions from utils import mediaFileMimeType @@ -2523,10 +2524,7 @@ class PubServer(BaseHTTPRequestHandler): return elif ('@' in searchStr or ('://' in searchStr and - ('/users/' in searchStr or - '/profile/' in searchStr or - '/accounts/' in searchStr or - '/channel/' in searchStr))): + hasUsersPath(searchStr))): # profile search nickname = getNicknameFromActor(actorStr) if not self.server.session: diff --git a/delete.py b/delete.py index 97b1aaa1c..240845a89 100644 --- a/delete.py +++ b/delete.py @@ -8,6 +8,7 @@ __status__ = "Production" import os from datetime import datetime +from utils import hasUsersPath from utils import getFullDomain from utils import removeIdEnding from utils import getNicknameFromActor @@ -139,10 +140,7 @@ def outboxDelete(baseDir: str, httpPrefix: str, if debug: print('DEBUG: c2s delete object is not a status') return - if '/users/' not in messageId and \ - '/accounts/' not in messageId and \ - '/channel/' not in messageId and \ - '/profile/' not in messageId: + if not hasUsersPath(messageId): if debug: print('DEBUG: c2s delete object has no nickname') return diff --git a/epicyon.py b/epicyon.py index 6b06ef495..3c111e4bf 100644 --- a/epicyon.py +++ b/epicyon.py @@ -47,6 +47,7 @@ from tests import testClientToServer from tests import runAllTests from auth import storeBasicCredentials from auth import createPassword +from utils import hasUsersPath from utils import getFullDomain from utils import setConfigParam from utils import getConfigParam @@ -1318,10 +1319,7 @@ if args.actor: for prefix in prefixes: args.actor = args.actor.replace(prefix, '') args.actor = args.actor.replace('/@', '/users/') - if '/users/' not in args.actor and \ - '/accounts/' not in args.actor and \ - '/channel/' not in args.actor and \ - '/profile/' not in args.actor: + if not hasUsersPath(args.actor): print('Expected actor format: ' + 'https://domain/@nick or https://domain/users/nick') sys.exit() @@ -1391,10 +1389,7 @@ if args.actor: personUrl = None if wfRequest.get('errors'): print('wfRequest error: ' + str(wfRequest['errors'])) - if '/users/' in args.actor or \ - '/accounts/' in args.actor or \ - '/profile/' in args.actor or \ - '/channel/' in args.actor: + if hasUsersPath(args.actor): personUrl = originalActor else: sys.exit() diff --git a/follow.py b/follow.py index 04a6a7db4..f3c209670 100644 --- a/follow.py +++ b/follow.py @@ -8,6 +8,7 @@ __status__ = "Production" from pprint import pprint import os +from utils import hasUsersPath from utils import getFullDomain from utils import isSystemAccount from utils import getFollowersList @@ -316,10 +317,7 @@ def _getNoOfFollows(baseDir: str, nickname: str, domain: str, ctr += 1 elif ((line.startswith('http') or line.startswith('dat')) and - ('/users/' in line or - '/profile/' in line or - '/accounts/' in line or - '/channel/' in line)): + hasUsersPath(line)): ctr += 1 return ctr @@ -438,10 +436,7 @@ def getFollowingFeed(baseDir: str, domain: str, port: int, path: str, following['orderedItems'].append(url) elif ((line.startswith('http') or line.startswith('dat')) and - ('/users/' in line or - '/profile/' in line or - '/accounts/' in line or - '/channel/' in line)): + hasUsersPath(line)): # https://domain/users/nickname pageCtr += 1 totalCtr += 1 @@ -616,10 +611,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, if debug: print('DEBUG: follow request has no actor') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: users/profile/accounts/channel missing from actor') return False @@ -641,10 +633,7 @@ def receiveFollowRequest(session, baseDir: str, httpPrefix: str, 'nickname. Assuming single user instance.') if not messageJson.get('to'): messageJson['to'] = messageJson['object'] - if '/users/' not in messageJson['object'] and \ - '/accounts/' not in messageJson['object'] and \ - '/channel/' not in messageJson['object'] and \ - '/profile/' not in messageJson['object']: + if not hasUsersPath(messageJson['object']): if debug: print('DEBUG: users/profile/channel/accounts ' + 'not found within object') diff --git a/inbox.py b/inbox.py index 680132308..96d78c4a1 100644 --- a/inbox.py +++ b/inbox.py @@ -10,6 +10,7 @@ import json import os import datetime import time +from utils import hasUsersPath from utils import validPostDate from utils import getFullDomain from utils import isEventPost @@ -604,10 +605,7 @@ def _receiveUndoFollow(session, baseDir: str, httpPrefix: str, if debug: print('DEBUG: follow request has no actor within object') return False - if '/users/' not in messageJson['object']['actor'] and \ - '/accounts/' not in messageJson['object']['actor'] and \ - '/channel/' not in messageJson['object']['actor'] and \ - '/profile/' not in messageJson['object']['actor']: + if not hasUsersPath(messageJson['object']['actor']): if debug: print('DEBUG: "users" or "profile" missing ' + 'from actor within object') @@ -668,10 +666,7 @@ def _receiveUndo(session, baseDir: str, httpPrefix: str, if debug: print('DEBUG: follow request has no actor') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor') return False @@ -859,10 +854,7 @@ def _receiveUpdate(recentPostsCache: {}, session, baseDir: str, if debug: print('DEBUG: ' + messageJson['type'] + ' object has no type') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor in ' + messageJson['type']) @@ -943,10 +935,7 @@ def _receiveLike(recentPostsCache: {}, if debug: print('DEBUG: ' + messageJson['type'] + ' has no "to" list') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor in ' + messageJson['type']) @@ -1014,10 +1003,7 @@ def _receiveUndoLike(recentPostsCache: {}, print('DEBUG: ' + messageJson['type'] + ' like object is not a string') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor in ' + messageJson['type'] + ' like') @@ -1219,10 +1205,7 @@ def _receiveDelete(session, handle: str, isGroup: bool, baseDir: str, if debug: print('DEBUG: ' + messageJson['type'] + ' has no "to" list') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: ' + '"users" or "profile" missing from actor in ' + @@ -1303,19 +1286,13 @@ def _receiveAnnounce(recentPostsCache: {}, if debug: print('DEBUG: ' + messageJson['type'] + ' has no "to" list') return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: ' + '"users" or "profile" missing from actor in ' + messageJson['type']) return False - if '/users/' not in messageJson['object'] and \ - '/accounts/' not in messageJson['object'] and \ - '/channel/' not in messageJson['object'] and \ - '/profile/' not in messageJson['object']: + if not hasUsersPath(messageJson['object']): if debug: print('DEBUG: ' + '"users", "channel" or "profile" missing in ' + @@ -1387,10 +1364,7 @@ def _receiveAnnounce(recentPostsCache: {}, if isinstance(attrib, str): lookupActor = attrib if lookupActor: - if '/users/' in lookupActor or \ - '/accounts/' in lookupActor or \ - '/channel/' in lookupActor or \ - '/profile/' in lookupActor: + if hasUsersPath(lookupActor): if '/statuses/' in lookupActor: lookupActor = lookupActor.split('/statuses/')[0] @@ -1439,10 +1413,7 @@ def _receiveUndoAnnounce(recentPostsCache: {}, return False if messageJson['object']['type'] != 'Announce': return False - if '/users/' not in messageJson['actor'] and \ - '/accounts/' not in messageJson['actor'] and \ - '/channel/' not in messageJson['actor'] and \ - '/profile/' not in messageJson['actor']: + if not hasUsersPath(messageJson['actor']): if debug: print('DEBUG: "users" or "profile" missing from actor in ' + messageJson['type'] + ' announce') @@ -1688,10 +1659,7 @@ def _obtainAvatarForReplyPost(session, baseDir: str, httpPrefix: str, if not isinstance(lookupActor, str): return - if not ('/users/' in lookupActor or - '/accounts/' in lookupActor or - '/channel/' in lookupActor or - '/profile/' in lookupActor): + if not hasUsersPath(lookupActor): return if '/statuses/' in lookupActor: diff --git a/like.py b/like.py index 29efeda3f..289c930c9 100644 --- a/like.py +++ b/like.py @@ -6,6 +6,7 @@ __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" +from utils import hasUsersPath from utils import getFullDomain from utils import removeIdEnding from utils import urlPermitted @@ -87,10 +88,7 @@ def _like(recentPostsCache: {}, likedPostNickname = getNicknameFromActor(actorLiked) likedPostDomain, likedPostPort = getDomainFromActor(actorLiked) else: - if '/users/' in objectUrl or \ - '/accounts/' in objectUrl or \ - '/channel/' in objectUrl or \ - '/profile/' in objectUrl: + if hasUsersPath(objectUrl): likedPostNickname = getNicknameFromActor(objectUrl) likedPostDomain, likedPostPort = getDomainFromActor(objectUrl) diff --git a/posts.py b/posts.py index 895c50c7e..c402e883c 100644 --- a/posts.py +++ b/posts.py @@ -30,6 +30,7 @@ from session import postJsonString from session import postImage from webfinger import webfingerHandle from httpsig import createSignedHeader +from utils import hasUsersPath from utils import validPostDate from utils import getFullDomain from utils import getFollowersList @@ -155,10 +156,7 @@ def getUserUrl(wfRequest: {}, sourceId=0) -> str: continue if link['type'] != 'application/activity+json': continue - if not ('/users/' in link['href'] or - '/accounts/' in link['href'] or - '/profile/' in link['href'] or - '/channel/' in link['href']): + if not hasUsersPath(link['href']): print('getUserUrl webfinger activity+json ' + 'contains single user instance actor ' + str(sourceId) + ' ' + str(link)) diff --git a/utils.py b/utils.py index ff89afd02..2bdcbff74 100644 --- a/utils.py +++ b/utils.py @@ -19,6 +19,16 @@ from calendar import monthrange from followingCalendar import addPersonToCalendar +def hasUsersPath(pathStr: str) -> bool: + """Whether there is a /users/ path (or equivalent) in the given string + """ + usersList = ('users', 'accounts', 'channel', 'profile') + for usersStr in usersList: + if '/' + usersStr + '/' in pathStr: + return True + return False + + def validPostDate(published: str, maxAgeDays=7) -> bool: """Returns true if the published date is recent and is not in the future """ diff --git a/webapp_profile.py b/webapp_profile.py index 5cc8866db..e89c4de32 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -8,6 +8,7 @@ __status__ = "Production" import os from pprint import pprint +from utils import hasUsersPath from utils import getFullDomain from utils import isDormant from utils import getNicknameFromActor @@ -61,11 +62,7 @@ def htmlProfileAfterSearch(cssCache: {}, defaultTimeline: str) -> str: """Show a profile page after a search for a fediverse address """ - if '/users/' in profileHandle or \ - '/accounts/' in profileHandle or \ - '/channel/' in profileHandle or \ - '/profile/' in profileHandle or \ - '/@' in profileHandle: + if hasUsersPath(profileHandle) or '/@' in profileHandle: searchNickname = getNicknameFromActor(profileHandle) searchDomain, searchPort = getDomainFromActor(profileHandle) else: From 0aa42a1d71694b627f96a2fd089f2d4e3f5f8772 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 10:58:50 +0000 Subject: [PATCH 02/31] Allow more time --- tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 5bd93560e..1c864100d 100644 --- a/tests.py +++ b/tests.py @@ -854,7 +854,7 @@ def testFollowBetweenServers(): True, __version__, False) print('sendResult: ' + str(sendResult)) - for t in range(10): + for t in range(16): if os.path.isfile(bobDir + '/accounts/bob@' + bobDomain + '/followers.txt'): if os.path.isfile(aliceDir + '/accounts/alice@' + From cfd1e7b24c44e1a8e3b313f722118d914099c245 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 12:48:50 +0000 Subject: [PATCH 03/31] Fix function tests --- daemon.py | 540 ++++++++++++++++++++++++++++++++++-------------------- tests.py | 49 ++++- 2 files changed, 377 insertions(+), 212 deletions(-) diff --git a/daemon.py b/daemon.py index 15bdd70db..a4c86ab89 100644 --- a/daemon.py +++ b/daemon.py @@ -581,7 +581,8 @@ class PubServer(BaseHTTPRequestHandler): def _set_headers_etag(self, mediaFilename: str, fileFormat: str, data, cookie: str, callingDomain: str) -> None: - self._set_headers_base(fileFormat, len(data), cookie, callingDomain) + datalen = len(data) + self._set_headers_base(fileFormat, datalen, cookie, callingDomain) self.send_header('Cache-Control', 'public, max-age=86400') etag = None if os.path.isfile(mediaFilename + '.etag'): @@ -665,7 +666,8 @@ class PubServer(BaseHTTPRequestHandler): msg = msg.encode('utf-8') self.send_response(httpCode) self.send_header('Content-Type', 'text/html; charset=utf-8') - self.send_header('Content-Length', str(len(msg))) + msgLenStr = str(len(msg)) + self.send_header('Content-Length', msgLenStr) self.send_header('X-Robots-Tag', 'noindex') self.end_headers() if not self._write(msg): @@ -743,7 +745,8 @@ class PubServer(BaseHTTPRequestHandler): return False msg = 'User-agent: *\nDisallow: /' msg = msg.encode('utf-8') - self._set_headers('text/plain; charset=utf-8', len(msg), + msglen = len(msg) + self._set_headers('text/plain; charset=utf-8', msglen, None, self.server.domainFull) self._write(msg) return True @@ -789,15 +792,16 @@ class PubServer(BaseHTTPRequestHandler): self.server.systemLanguage, self.server.projectVersion) msg = json.dumps(instanceJson).encode('utf-8') + msglen = len(msg) if self._hasAccept(callingDomain): if 'application/ld+json' in self.headers['Accept']: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) else: - self._set_headers('application/json', len(msg), + self._set_headers('application/json', msglen, None, callingDomain) else: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) self._write(msg) print('instance metadata sent') @@ -810,15 +814,16 @@ class PubServer(BaseHTTPRequestHandler): # information about the interests of a small number of accounts msg = json.dumps(['mastodon.social', self.server.domainFull]).encode('utf-8') + msglen = len(msg) if self._hasAccept(callingDomain): if 'application/ld+json' in self.headers['Accept']: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) else: - self._set_headers('application/json', len(msg), + self._set_headers('application/json', msglen, None, callingDomain) else: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) self._write(msg) print('instance peers metadata sent') @@ -826,15 +831,16 @@ class PubServer(BaseHTTPRequestHandler): if self.path.startswith('/api/v1/instance/activity'): # This is just a dummy result. msg = json.dumps([]).encode('utf-8') + msglen = len(msg) if self._hasAccept(callingDomain): if 'application/ld+json' in self.headers['Accept']: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) else: - self._set_headers('application/json', len(msg), + self._set_headers('application/json', msglen, None, callingDomain) else: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) self._write(msg) print('instance activity metadata sent') @@ -852,15 +858,16 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion) if info: msg = json.dumps(info).encode('utf-8') + msglen = len(msg) if self._hasAccept(callingDomain): if 'application/ld+json' in self.headers['Accept']: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) else: - self._set_headers('application/json', len(msg), + self._set_headers('application/json', msglen, None, callingDomain) else: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) self._write(msg) print('nodeinfo sent') @@ -891,7 +898,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.domainFull) if wfResult: msg = wfResult.encode('utf-8') - self._set_headers('application/xrd+xml', len(msg), + msglen = len(msg) + self._set_headers('application/xrd+xml', msglen, None, callingDomain) self._write(msg) return True @@ -912,15 +920,16 @@ class PubServer(BaseHTTPRequestHandler): self.server.domainFull) if wfResult: msg = json.dumps(wfResult).encode('utf-8') + msglen = len(msg) if self._hasAccept(callingDomain): if 'application/ld+json' in self.headers['Accept']: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) else: - self._set_headers('application/json', len(msg), + self._set_headers('application/json', msglen, None, callingDomain) else: - self._set_headers('application/ld+json', len(msg), + self._set_headers('application/ld+json', msglen, None, callingDomain) self._write(msg) return True @@ -936,7 +945,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.port, self.server.debug) if wfResult: msg = json.dumps(wfResult).encode('utf-8') - self._set_headers('application/jrd+json', len(msg), + msglen = len(msg) + self._set_headers('application/jrd+json', msglen, None, callingDomain) self._write(msg) else: @@ -1310,8 +1320,9 @@ class PubServer(BaseHTTPRequestHandler): msg = \ htmlSuspended(self.server.cssCache, baseDir).encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -1484,8 +1495,9 @@ class PubServer(BaseHTTPRequestHandler): baseDir, httpPrefix, nickname) msg = msg.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -1736,9 +1748,10 @@ class PubServer(BaseHTTPRequestHandler): chooserNickname, domain, handle, petname) - self._redirect_headers(usersPath + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + usersPathStr = \ + usersPath + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(usersPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -1755,9 +1768,10 @@ class PubServer(BaseHTTPRequestHandler): chooserNickname, domain, handle, personNotes) - self._redirect_headers(usersPath + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + usersPathStr = \ + usersPath + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(usersPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -1782,9 +1796,10 @@ class PubServer(BaseHTTPRequestHandler): domain, optionsNickname, optionsDomainFull) - self._redirect_headers(usersPath + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + usersPathStr = \ + usersPath + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(usersPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -1814,9 +1829,10 @@ class PubServer(BaseHTTPRequestHandler): if noNewswireFile: noNewswireFile.write('\n') noNewswireFile.close() - self._redirect_headers(usersPath + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + usersPathStr = \ + usersPath + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(usersPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -1847,9 +1863,10 @@ class PubServer(BaseHTTPRequestHandler): if modNewswireFile: modNewswireFile.write('\n') modNewswireFile.close() - self._redirect_headers(usersPath + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + usersPathStr = \ + usersPath + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(usersPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -1876,7 +1893,8 @@ class PubServer(BaseHTTPRequestHandler): usersPath, optionsActor, optionsAvatarUrl).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -1894,7 +1912,8 @@ class PubServer(BaseHTTPRequestHandler): usersPath, optionsActor, optionsAvatarUrl).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -1911,7 +1930,8 @@ class PubServer(BaseHTTPRequestHandler): usersPath, optionsActor, optionsAvatarUrl).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -1936,7 +1956,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline, self.server.newswire, self.server.themeName).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -1958,7 +1979,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.port, optionsActor, self.server.debug).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -1982,9 +2004,10 @@ class PubServer(BaseHTTPRequestHandler): thisActor = 'http://' + onionDomain + usersPath elif (callingDomain.endswith('.i2p') and i2pDomain): thisActor = 'http://' + i2pDomain + usersPath - self._redirect_headers(thisActor + '/' + - self.server.defaultTimeline + - '?page='+str(pageNumber), cookie, + actorPathStr = \ + thisActor + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -2004,9 +2027,10 @@ class PubServer(BaseHTTPRequestHandler): thisActor = 'http://' + onionDomain + usersPath elif (callingDomain.endswith('.i2p') and i2pDomain): thisActor = 'http://' + i2pDomain + usersPath - self._redirect_headers(thisActor + '/' + - self.server.defaultTimeline + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + thisActor + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -2030,7 +2054,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline, self.server.newswire, self.server.themeName).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.POSTbusy = False @@ -2468,8 +2493,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.showPublishedDateOnly) if hashtagStr: msg = hashtagStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2487,8 +2513,9 @@ class PubServer(BaseHTTPRequestHandler): 64) if skillStr: msg = skillStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2517,8 +2544,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.showPublishedDateOnly) if historyStr: msg = historyStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2560,8 +2588,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline) if profileStr: msg = profileStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2589,8 +2618,9 @@ class PubServer(BaseHTTPRequestHandler): searchStr) if emojiStr: msg = emojiStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2607,8 +2637,9 @@ class PubServer(BaseHTTPRequestHandler): actorStr, callingDomain) if sharedItemsStr: msg = sharedItemsStr.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) self.server.POSTbusy = False return @@ -2647,9 +2678,10 @@ class PubServer(BaseHTTPRequestHandler): actor = 'http://' + onionDomain + usersPath elif (callingDomain.endswith('.i2p') and i2pDomain): actor = 'http://' + i2pDomain + usersPath - self._redirect_headers(actor + '/' + - self.server.defaultTimeline + - '?page=' + str(pageNumber), + actorPathStr = \ + actor + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -2699,9 +2731,10 @@ class PubServer(BaseHTTPRequestHandler): actor = 'http://' + onionDomain + usersPath elif (callingDomain.endswith('.i2p') and i2pDomain): actor = 'http://' + i2pDomain + usersPath - self._redirect_headers(actor + '/' + - self.server.defaultTimeline + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + actor + '/' + self.server.defaultTimeline + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False return @@ -2928,8 +2961,9 @@ class PubServer(BaseHTTPRequestHandler): self._redirect_headers(originPathStr + '/outbox', cookie, callingDomain) else: - self._redirect_headers(originPathStr + '/outbox?page=' + - str(pageNumber), + pageNumberStr = str(pageNumber) + actorPathStr = originPathStr + '/outbox?page=' + pageNumberStr + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False @@ -4698,7 +4732,8 @@ class PubServer(BaseHTTPRequestHandler): } msg = json.dumps(manifest, ensure_ascii=False).encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) if self.server.debug: @@ -4860,7 +4895,8 @@ class PubServer(BaseHTTPRequestHandler): True) if msg is not None: msg = msg.encode('utf-8') - self._set_headers('text/xml', len(msg), + msglen = len(msg) + self._set_headers('text/xml', msglen, None, callingDomain) self._write(msg) if debug: @@ -4921,7 +4957,8 @@ class PubServer(BaseHTTPRequestHandler): 'Site', translate) + msg + rss2Footer() msg = msg.encode('utf-8') - self._set_headers('text/xml', len(msg), + msglen = len(msg) + self._set_headers('text/xml', msglen, None, callingDomain) self._write(msg) if debug: @@ -4960,7 +4997,8 @@ class PubServer(BaseHTTPRequestHandler): 'Newswire', self.server.translate) if msg: msg = msg.encode('utf-8') - self._set_headers('text/xml', len(msg), + msglen = len(msg) + self._set_headers('text/xml', msglen, None, callingDomain) self._write(msg) if debug: @@ -4995,7 +5033,8 @@ class PubServer(BaseHTTPRequestHandler): getHashtagCategoriesFeed(baseDir, hashtagCategories) if msg: msg = msg.encode('utf-8') - self._set_headers('text/xml', len(msg), + msglen = len(msg) + self._set_headers('text/xml', msglen, None, callingDomain) self._write(msg) if debug: @@ -5039,8 +5078,9 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInRSSFeed, 1) if msg is not None: msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('text/plain; charset=utf-8', - len(msg), None, callingDomain) + msglen, None, callingDomain) self._write(msg) if self.server.debug: print('Sent rss3 feed: ' + @@ -5126,7 +5166,8 @@ class PubServer(BaseHTTPRequestHandler): emailAddress, self.server.dormantMonths, backToPath).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -5243,8 +5284,9 @@ class PubServer(BaseHTTPRequestHandler): return if self.server.iconsCache.get(mediaStr): mediaBinary = self.server.iconsCache[mediaStr] + mimeTypeStr = mediaFileMimeType(mediaFilename) self._set_headers_etag(mediaFilename, - mediaFileMimeType(mediaFilename), + mimeTypeStr, mediaBinary, None, callingDomain) self._write(mediaBinary) @@ -5314,7 +5356,8 @@ class PubServer(BaseHTTPRequestHandler): print('BLOCK: hashtag #' + hashtag) msg = htmlHashtagBlocked(self.server.cssCache, baseDir, self.server.translate).encode('utf-8') - self._login_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._login_headers('text/html', msglen, callingDomain) self._write(msg) self.server.GETbusy = False return @@ -5340,7 +5383,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.showPublishedDateOnly) if hashtagStr: msg = hashtagStr.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -5394,7 +5438,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.YTReplacementDomain) if hashtagStr: msg = hashtagStr.encode('utf-8') - self._set_headers('text/xml', len(msg), + msglen = len(msg) + self._set_headers('text/xml', msglen, cookie, callingDomain) self._write(msg) else: @@ -5459,8 +5504,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and i2pDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -5504,10 +5551,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + - timelineStr + '?page=' + - str(pageNumber) + - timelineBookmark, cookie, callingDomain) + actorPathStr = \ + actorAbsolute + '/' + timelineStr + '?page=' + \ + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'emoji search shown done', 'show announce') @@ -5555,9 +5602,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and i2pDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + - timelineStr + '?page=' + - str(pageNumber), cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + '?page=' + \ + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -5597,10 +5645,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and i2pDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + - timelineStr + '?page=' + - str(pageNumber) + - timelineBookmark, cookie, callingDomain) + actorPathStr = \ + actorAbsolute + '/' + timelineStr + '?page=' + \ + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'show announce done', 'unannounce') @@ -5850,9 +5898,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and i2pDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber) + - timelineBookmark, cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -5904,9 +5953,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and i2pDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber) + - timelineBookmark, cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'follow deny done', @@ -5955,8 +6005,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif (callingDomain.endswith('.i2p') and onionDomain): actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -6007,9 +6059,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber) + - timelineBookmark, cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'like shown done', @@ -6059,8 +6112,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -6101,9 +6156,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber) + - timelineBookmark, cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'unlike shown done', @@ -6152,8 +6208,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) return if not self.server.session: @@ -6194,9 +6252,10 @@ class PubServer(BaseHTTPRequestHandler): actorAbsolute = 'http://' + onionDomain + actor elif callingDomain.endswith('.i2p') and i2pDomain: actorAbsolute = 'http://' + i2pDomain + actor - self._redirect_headers(actorAbsolute + '/' + timelineStr + - '?page=' + str(pageNumber) + - timelineBookmark, cookie, + actorPathStr = \ + actorAbsolute + '/' + timelineStr + \ + '?page=' + str(pageNumber) + timelineBookmark + self._redirect_headers(actorPathStr, cookie, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, 'bookmark shown done', @@ -6284,7 +6343,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.YTReplacementDomain, self.server.showPublishedDateOnly) if deleteStr: - self._set_headers('text/html', len(deleteStr), + deleteStrLen = len(deleteStr) + self._set_headers('text/html', deleteStrLen, cookie, callingDomain) self._write(deleteStr.encode('utf-8')) self.server.GETbusy = False @@ -6485,7 +6545,8 @@ class PubServer(BaseHTTPRequestHandler): ytDomain, self.server.showPublishedDateOnly) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -6493,7 +6554,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(repliesJson, ensure_ascii=False) msg = msg.encode('utf-8') protocolStr = 'application/json' - self._set_headers(protocolStr, len(msg), None, + msglen = len(msg) + self._set_headers(protocolStr, msglen, None, callingDomain) self._write(msg) else: @@ -6567,7 +6629,8 @@ class PubServer(BaseHTTPRequestHandler): ytDomain, self.server.showPublishedDateOnly) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, @@ -6580,7 +6643,8 @@ class PubServer(BaseHTTPRequestHandler): ensure_ascii=False) msg = msg.encode('utf-8') protocolStr = 'application/json' - self._set_headers(protocolStr, len(msg), + msglen = len(msg) + self._set_headers(protocolStr, msglen, None, callingDomain) self._write(msg) else: @@ -6652,7 +6716,8 @@ class PubServer(BaseHTTPRequestHandler): actorJson['roles'], None, None) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -6663,7 +6728,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(actorJson['roles'], ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -6733,7 +6799,8 @@ class PubServer(BaseHTTPRequestHandler): actorJson['skills'], None, None) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, @@ -6745,8 +6812,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(actorJson['skills'], ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), None, + msglen, None, callingDomain) self._write(msg) else: @@ -6853,7 +6921,8 @@ class PubServer(BaseHTTPRequestHandler): ytDomain, showPublishedDateOnly) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -6861,8 +6930,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(postJsonObject, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) else: @@ -6965,7 +7035,8 @@ class PubServer(BaseHTTPRequestHandler): ytDomain, showPublishedDateOnly) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, @@ -6978,8 +7049,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(postJsonObject, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) else: @@ -7101,7 +7173,8 @@ class PubServer(BaseHTTPRequestHandler): if msg: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) @@ -7114,7 +7187,8 @@ class PubServer(BaseHTTPRequestHandler): # there is already the authorization check msg = json.dumps(inboxFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7219,7 +7293,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7230,8 +7305,9 @@ class PubServer(BaseHTTPRequestHandler): # there is already the authorization check msg = json.dumps(inboxDMFeed, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7336,7 +7412,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7348,7 +7425,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(inboxRepliesFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7454,7 +7532,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7466,7 +7545,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(inboxMediaFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7572,7 +7652,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7584,8 +7665,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(inboxBlogsFeed, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7699,7 +7781,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7711,8 +7794,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(inboxNewsFeed, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7822,7 +7906,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -7834,8 +7919,9 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(inboxFeaturesFeed, ensure_ascii=False) msg = msg.encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -7906,7 +7992,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.publishButtonAtTop, authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8008,7 +8095,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8020,7 +8108,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(bookmarksFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -8129,7 +8218,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8141,7 +8231,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(eventsFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -8242,7 +8333,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8253,7 +8345,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(outboxFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -8345,7 +8438,8 @@ class PubServer(BaseHTTPRequestHandler): authorized, moderationActionStr, self.server.themeName) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8357,7 +8451,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(moderationFeed, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) self.server.GETbusy = False @@ -8443,7 +8538,8 @@ class PubServer(BaseHTTPRequestHandler): shares, pageNumber, sharesPerPage) msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8456,7 +8552,8 @@ class PubServer(BaseHTTPRequestHandler): msg = json.dumps(shares, ensure_ascii=False) msg = msg.encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -8536,8 +8633,9 @@ class PubServer(BaseHTTPRequestHandler): following, pageNumber, followsPerPage).encode('utf-8') + msglen = len(msg) self._set_headers('text/html', - len(msg), cookie, callingDomain) + msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8548,7 +8646,8 @@ class PubServer(BaseHTTPRequestHandler): if self._fetchAuthenticated(): msg = json.dumps(following, ensure_ascii=False).encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -8629,7 +8728,8 @@ class PubServer(BaseHTTPRequestHandler): followers, pageNumber, followsPerPage).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -8641,7 +8741,8 @@ class PubServer(BaseHTTPRequestHandler): if self._fetchAuthenticated(): msg = json.dumps(followers, ensure_ascii=False).encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -8695,7 +8796,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName, self.server.dormantMonths, None, None).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8705,7 +8807,8 @@ class PubServer(BaseHTTPRequestHandler): if self._fetchAuthenticated(): msg = json.dumps(getPerson, ensure_ascii=False).encode('utf-8') - self._set_headers('application/json', len(msg), + msglen = len(msg) + self._set_headers('application/json', msglen, None, callingDomain) self._write(msg) else: @@ -8760,7 +8863,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed, pageNumber) if msg is not None: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -8850,7 +8954,8 @@ class PubServer(BaseHTTPRequestHandler): time.sleep(1) tries += 1 msg = css.encode('utf-8') - self._set_headers('text/css', len(msg), + msglen = len(msg) + self._set_headers('text/css', msglen, None, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -9175,7 +9280,8 @@ class PubServer(BaseHTTPRequestHandler): 'calendar delete shown') return True msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -9222,7 +9328,8 @@ class PubServer(BaseHTTPRequestHandler): self._404() self.server.GETbusy = False return True - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -9248,7 +9355,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline, self.server.themeName).encode('utf-8') if msg: - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -9273,7 +9381,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline, theme).encode('utf-8') if msg: - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -9298,7 +9407,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.defaultTimeline, self.server.themeName).encode('utf-8') if msg: - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -9332,7 +9442,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, postUrl).encode('utf-8') if msg: - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) else: @@ -9371,7 +9482,8 @@ class PubServer(BaseHTTPRequestHandler): # postUrl) if msg: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -9408,15 +9520,15 @@ class PubServer(BaseHTTPRequestHandler): return self._benchmarkGETtimings(GETstartTime, GETtimings, - 'start', '_nodeinfo(callingDomain)') + 'start', '_nodeinfo[callingDomain]') # minimal mastodon api if self._mastoApi(callingDomain): return self._benchmarkGETtimings(GETstartTime, GETtimings, - '_nodeinfo(callingDomain)', - '_mastoApi(callingDomain)') + '_nodeinfo[callingDomain]', + '_mastoApi[callingDomain]') if self.path == '/logout': if not self.server.newsInstance: @@ -9424,7 +9536,8 @@ class PubServer(BaseHTTPRequestHandler): htmlLogin(self.server.cssCache, self.server.translate, self.server.baseDir, False).encode('utf-8') - self._logout_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._logout_headers('text/html', msglen, callingDomain) self._write(msg) else: if callingDomain.endswith('.onion') and \ @@ -9446,12 +9559,12 @@ class PubServer(BaseHTTPRequestHandler): '/users/news', None, callingDomain) self._benchmarkGETtimings(GETstartTime, GETtimings, - '_nodeinfo(callingDomain)', + '_nodeinfo[callingDomain]', 'logout') return self._benchmarkGETtimings(GETstartTime, GETtimings, - '_nodeinfo(callingDomain)', + '_nodeinfo[callingDomain]', 'show logout') # replace https://domain/@nick with https://domain/users/nick @@ -9664,7 +9777,8 @@ class PubServer(BaseHTTPRequestHandler): maxPostsInBlogsFeed) if msg is not None: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -9709,8 +9823,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix) msg = json.dumps(devJson, ensure_ascii=False).encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -9758,7 +9873,8 @@ class PubServer(BaseHTTPRequestHandler): postJsonObject) if msg is not None: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -9795,7 +9911,8 @@ class PubServer(BaseHTTPRequestHandler): self._redirect_headers(actor + '/tlshares', cookie, callingDomain) return - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -9824,7 +9941,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.httpPrefix, self.server.domainFull) msg = msg.encode('utf-8') - self._login_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._login_headers('text/html', msglen, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, 'blog post 2 done', @@ -9847,7 +9965,8 @@ class PubServer(BaseHTTPRequestHandler): return msg = htmlFollowingList(self.server.cssCache, self.server.baseDir, followingFilename) - self._login_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._login_headers('text/html', msglen, callingDomain) self._write(msg.encode('utf-8')) self._benchmarkGETtimings(GETstartTime, GETtimings, 'terms of service done', @@ -9879,7 +9998,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.domainFull, self.server.onionDomain) msg = msg.encode('utf-8') - self._login_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._login_headers('text/html', msglen, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, 'following accounts done', @@ -10032,8 +10152,9 @@ class PubServer(BaseHTTPRequestHandler): time.sleep(1) tries += 1 if mediaBinary: + mimeTypeStr = mediaFileMimeType(iconFilename) self._set_headers_etag(iconFilename, - mediaFileMimeType(iconFilename), + mimeTypeStr, mediaBinary, cookie, callingDomain) self._write(mediaBinary) @@ -10219,7 +10340,8 @@ class PubServer(BaseHTTPRequestHandler): msg = htmlLogin(self.server.cssCache, self.server.translate, self.server.baseDir).encode('utf-8') - self._login_headers('text/html', len(msg), callingDomain) + msglen = len(msg) + self._login_headers('text/html', msglen, callingDomain) self._write(msg) self.server.GETbusy = False self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -10291,7 +10413,8 @@ class PubServer(BaseHTTPRequestHandler): iconsAsButtons, defaultTimeline, self.server.themeName).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -10323,7 +10446,8 @@ class PubServer(BaseHTTPRequestHandler): iconsAsButtons, defaultTimeline, self.server.themeName).encode('utf-8') - self._set_headers('text/html', len(msg), cookie, callingDomain) + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False return @@ -10392,7 +10516,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.domain, self.server.defaultTimeline, self.server.themeName).encode('utf-8') - self._set_headers('text/html', len(msg), cookie, callingDomain) + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -10409,7 +10534,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName) if msg: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), cookie, callingDomain) + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -10430,7 +10556,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.baseDir, self.path, self.server.httpPrefix, self.server.domainFull).encode('utf-8') - self._set_headers('text/html', len(msg), cookie, callingDomain) + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -10470,7 +10597,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate, self.server.baseDir, self.path).encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -10860,7 +10988,8 @@ class PubServer(BaseHTTPRequestHandler): postUrl) if msg: msg = msg.encode('utf-8') - self._set_headers('text/html', len(msg), + msglen = len(msg) + self._set_headers('text/html', msglen, cookie, callingDomain) self._write(msg) self.server.GETbusy = False @@ -11236,8 +11365,9 @@ class PubServer(BaseHTTPRequestHandler): searchHandle, self.server.debug) msg = msg.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) return @@ -11269,8 +11399,9 @@ class PubServer(BaseHTTPRequestHandler): searchHandle, self.server.debug) msg = msg.encode('utf-8') + msglen = len(msg) self._login_headers('text/html', - len(msg), callingDomain) + msglen, callingDomain) self._write(msg) return @@ -11461,8 +11592,9 @@ class PubServer(BaseHTTPRequestHandler): contentJson = json.loads(content) msg = json.dumps(contentJson, ensure_ascii=False).encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -11733,8 +11865,9 @@ class PubServer(BaseHTTPRequestHandler): self.server.themeName) if messageJson: messageJson = messageJson.encode('utf-8') + messageJsonLen = len(messageJson) self._set_headers('text/html', - len(messageJson), + messageJsonLen, cookie, callingDomain) self._write(messageJson) return 1 @@ -12337,8 +12470,9 @@ class PubServer(BaseHTTPRequestHandler): msg = \ json.dumps(devicesList, ensure_ascii=False).encode('utf-8') + msglen = len(msg) self._set_headers('application/json', - len(msg), + msglen, None, callingDomain) self._write(msg) return True @@ -12774,26 +12908,26 @@ class PubServer(BaseHTTPRequestHandler): if callingDomain.endswith('.onion') and \ self.server.onionDomain: - self._redirect_headers('http://' + - self.server.onionDomain + - '/users/' + nickname + - '/' + postRedirect + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + 'http://' + self.server.onionDomain + \ + '/users/' + nickname + '/' + postRedirect + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) elif (callingDomain.endswith('.i2p') and self.server.i2pDomain): - self._redirect_headers('http://' + - self.server.i2pDomain + - '/users/' + nickname + - '/' + postRedirect + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + 'http://' + self.server.i2pDomain + \ + '/users/' + nickname + '/' + postRedirect + \ + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) else: - self._redirect_headers(self.server.httpPrefix + '://' + - self.server.domainFull + - '/users/' + nickname + - '/' + postRedirect + - '?page=' + str(pageNumber), cookie, + actorPathStr = \ + self.server.httpPrefix + '://' + \ + self.server.domainFull + '/users/' + nickname + \ + '/' + postRedirect + '?page=' + str(pageNumber) + self._redirect_headers(actorPathStr, cookie, callingDomain) self.server.POSTbusy = False return diff --git a/tests.py b/tests.py index 1c864100d..7444f724c 100644 --- a/tests.py +++ b/tests.py @@ -2613,9 +2613,10 @@ def getFunctionCalls(name: str, lines: [], startLineCtr: int, callsFunctions = [] functionContentStr = '' for lineCtr in range(startLineCtr + 1, len(lines)): - if lines[lineCtr].startswith('def '): + lineStr = lines[lineCtr].strip() + if lineStr.startswith('def '): break - if lines[lineCtr].startswith('class '): + if lineStr.startswith('class '): break functionContentStr += lines[lineCtr] for funcName, properties in functionProperties.items(): @@ -2635,14 +2636,14 @@ def functionArgsMatch(callArgs: [], funcArgs: []): for a in callArgs: if a == 'self': continue - if '=' not in a: + if '=' not in a or a.startswith("'"): callArgsCtr += 1 funcArgsCtr = 0 for a in funcArgs: if a == 'self': continue - if '=' not in a: + if '=' not in a or a.startswith("'"): funcArgsCtr += 1 return callArgsCtr >= funcArgsCtr @@ -2670,7 +2671,7 @@ def testFunctions(): lines = f.readlines() modules[modName]['lines'] = lines for line in lines: - if not line.startswith('def '): + if not line.strip().startswith('def '): continue methodName = line.split('def ', 1)[1].split('(')[0] methodArgs = \ @@ -2694,7 +2695,9 @@ def testFunctions(): 'pyjsonld' ] excludeFuncs = [ - 'link' + 'link', + 'set', + 'get' ] # which modules is each function used within? for modName, modProperties in modules.items(): @@ -2702,7 +2705,11 @@ def testFunctions(): for name, properties in functionProperties.items(): lineCtr = 0 for line in modules[modName]['lines']: - if line.startswith('def '): + lineStr = line.strip() + if lineStr.startswith('def '): + lineCtr += 1 + continue + if lineStr.startswith('class '): lineCtr += 1 continue if name + '(' in line: @@ -2735,7 +2742,22 @@ def testFunctions(): # don't check these functions, because they are procedurally called exclusions = [ + 'do_GET', + 'do_POST', + 'do_HEAD', + '__run', + 'globaltrace', + 'localtrace', + 'kill', + 'clone', + 'unregister_rdf_parser', 'set_document_loader', + 'has_property', + 'has_value', + 'add_value', + 'get_values', + 'remove_property', + 'remove_value', 'normalize', 'get_document_loader', 'runInboxQueueWatchdog', @@ -2764,17 +2786,26 @@ def testFunctions(): 'setOrganizationScheme' ] excludeImports = [ - 'link' + 'link', + 'start' ] excludeLocal = [ 'pyjsonld', 'daemon', 'tests' ] + excludeMods = [ + 'pyjsonld' + ] # check that functions are called somewhere for name, properties in functionProperties.items(): + if name.startswith('__'): + if name.endswith('__'): + continue if name in exclusions: continue + if properties['module'] in excludeMods: + continue isLocalFunction = False if not properties['calledInModule']: print('function ' + name + @@ -2819,7 +2850,7 @@ def testFunctions(): for modName, modProperties in modules.items(): lineCtr = 0 for line in modules[modName]['lines']: - if line.startswith('def '): + if line.strip().startswith('def '): name = line.split('def ')[1].split('(')[0] callsList = \ getFunctionCalls(name, modules[modName]['lines'], From 72b07306f9c0e2c03cf2d516c47d9710e4b113bf Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 13:09:44 +0000 Subject: [PATCH 04/31] Change dot command --- tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 7444f724c..4a2e2144c 100644 --- a/tests.py +++ b/tests.py @@ -2883,7 +2883,7 @@ def testFunctions(): fp.write(callGraphStr) print('Call graph saved to epicyon.dot') print('Convert to image with: ' + - 'dot -Tjpg epicyon.dot -o epicyon_diagram.jpg') + 'sfdp -x -Goverlap=scale -Tx11 epicyon.dot') def runAllTests(): From 76a634b9dd1403f11ce11e05d89d7a51cc7dba77 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 13:41:54 +0000 Subject: [PATCH 05/31] sfdp settings --- tests.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests.py b/tests.py index 4a2e2144c..fdf111f4f 100644 --- a/tests.py +++ b/tests.py @@ -2865,10 +2865,12 @@ def testFunctions(): callGraphStr += ' subgraph cluster_' + modName + ' {\n' callGraphStr += ' label = "' + modName + '";\n' callGraphStr += ' node [style=filled];\n' - callGraphStr += ' ' + moduleFunctionsStr = '' for name in modProperties['functions']: - callGraphStr += '"' + name + '" ' - callGraphStr += ';\n' + if name not in excludeFuncs: + moduleFunctionsStr += '"' + name + '" ' + if moduleFunctionsStr: + callGraphStr += ' ' + moduleFunctionsStr + ';\n' callGraphStr += ' color=blue;\n' callGraphStr += ' }\n\n' @@ -2876,14 +2878,16 @@ def testFunctions(): if not properties['calls']: continue for calledFunc in properties['calls']: - callGraphStr += ' "' + name + '" -> "' + calledFunc + '";\n' + if calledFunc not in excludeFuncs: + callGraphStr += ' "' + name + '" -> "' + calledFunc + '";\n' callGraphStr += '\n}\n' with open('epicyon.dot', 'w+') as fp: fp.write(callGraphStr) print('Call graph saved to epicyon.dot') print('Convert to image with: ' + - 'sfdp -x -Goverlap=scale -Tx11 epicyon.dot') + 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + + '-Gsep=+120 -Tx11 epicyon.dot') def runAllTests(): From 549faf8b8507230783184850e2ce91644b8b3b1a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 14:19:06 +0000 Subject: [PATCH 06/31] Remove unit tests from dot diagram --- tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests.py b/tests.py index fdf111f4f..ec4a2b830 100644 --- a/tests.py +++ b/tests.py @@ -2867,6 +2867,8 @@ def testFunctions(): callGraphStr += ' node [style=filled];\n' moduleFunctionsStr = '' for name in modProperties['functions']: + if name.startswith('test'): + continue if name not in excludeFuncs: moduleFunctionsStr += '"' + name + '" ' if moduleFunctionsStr: @@ -2878,6 +2880,8 @@ def testFunctions(): if not properties['calls']: continue for calledFunc in properties['calls']: + if calledFunc.startswith('test'): + continue if calledFunc not in excludeFuncs: callGraphStr += ' "' + name + '" -> "' + calledFunc + '";\n' From d3261de520c231ac0de864dfb3d8aa221c4a5a1f Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 16:19:18 +0000 Subject: [PATCH 07/31] Plot modules call graph --- tests.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index ec4a2b830..9519c743e 100644 --- a/tests.py +++ b/tests.py @@ -2846,7 +2846,7 @@ def testFunctions(): assert False print('Function: ' + name + ' ✓') - print('Constructing call graph') + print('Constructing function call graph') for modName, modProperties in modules.items(): lineCtr = 0 for line in modules[modName]['lines']: @@ -2856,8 +2856,34 @@ def testFunctions(): getFunctionCalls(name, modules[modName]['lines'], lineCtr, functionProperties) functionProperties[name]['calls'] = callsList.copy() + # keep track of which module calls which other module + for fn in callsList: + modCall = functionProperties[fn]['module'] + if modCall != modName: + if modules[modName].get('calls'): + if modCall not in modules[modName]['calls']: + modules[modName]['calls'].append(modCall) + else: + modules[modName]['calls'] = [modCall] lineCtr += 1 + callGraphStr = 'digraph EpicyonModules {\n\n' + callGraphStr += ' graph [fontsize=10 fontname="Verdana" compound=true];\n' + callGraphStr += ' node [shape=record fontsize=10 fontname="Verdana"];\n\n' + for modName, modProperties in modules.items(): + if not modProperties.get('calls'): + continue + for modCall in modProperties['calls']: + callGraphStr += ' "' + modName + '" -> "' + modCall + '";\n' + callGraphStr += '\n}\n' + with open('epicyon_modules.dot', 'w+') as fp: + fp.write(callGraphStr) + print('Modules call graph saved to epicyon_modules.dot') + print('Plot using: ' + + 'sfdp -x -Goverlap=false -Goverlap_scaling=2 ' + + '-Gsep=+100 -Tx11 epicyon_modules.dot') + callGraphStr = 'digraph Epicyon {\n\n' + callGraphStr += ' size="8,6"; ratio=fill;\n' callGraphStr += ' graph [fontsize=10 fontname="Verdana" compound=true];\n' callGraphStr += ' node [shape=record fontsize=10 fontname="Verdana"];\n\n' @@ -2889,7 +2915,7 @@ def testFunctions(): with open('epicyon.dot', 'w+') as fp: fp.write(callGraphStr) print('Call graph saved to epicyon.dot') - print('Convert to image with: ' + + print('Plot using: ' + 'sfdp -x -Goverlap=prism -Goverlap_scaling=8 ' + '-Gsep=+120 -Tx11 epicyon.dot') From f7d98640935bbcd19466914cc25fb3f4de76e8ea Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 17:56:25 +0000 Subject: [PATCH 08/31] Module colors --- tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests.py b/tests.py index 9519c743e..35e629e0e 100644 --- a/tests.py +++ b/tests.py @@ -2847,6 +2847,7 @@ def testFunctions(): print('Function: ' + name + ' ✓') print('Constructing function call graph') + maxModuleCalls = 1 for modName, modProperties in modules.items(): lineCtr = 0 for line in modules[modName]['lines']: @@ -2863,12 +2864,30 @@ def testFunctions(): if modules[modName].get('calls'): if modCall not in modules[modName]['calls']: modules[modName]['calls'].append(modCall) + if len(modules[modName]['calls']) > \ + maxModuleCalls: + maxModuleCalls = \ + len(modules[modName]['calls']) else: modules[modName]['calls'] = [modCall] lineCtr += 1 callGraphStr = 'digraph EpicyonModules {\n\n' callGraphStr += ' graph [fontsize=10 fontname="Verdana" compound=true];\n' callGraphStr += ' node [shape=record fontsize=10 fontname="Verdana"];\n\n' + # colors of modules nodes + for modName, modProperties in modules.items(): + if not modProperties.get('calls'): + callGraphStr += ' "' + modName + \ + '" [fillcolor = yellow style=filled];\n' + continue + if len(modProperties['calls']) < int(maxModuleCalls / 4): + callGraphStr += ' "' + modName + \ + '" [fillcolor = orange style=filled];\n' + else: + callGraphStr += ' "' + modName + \ + '" [fillcolor = red style=filled];\n' + callGraphStr += '\n' + # connections between modules for modName, modProperties in modules.items(): if not modProperties.get('calls'): continue From db875e375c7a1a47fdcb85742126fbc9c9e235d4 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 18:15:30 +0000 Subject: [PATCH 09/31] Link color --- tests.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 35e629e0e..bc3b0d312 100644 --- a/tests.py +++ b/tests.py @@ -2847,9 +2847,20 @@ def testFunctions(): print('Function: ' + name + ' ✓') print('Constructing function call graph') + moduleColors = ('red', 'green', 'yellow', 'orange', 'purple', 'cyan', + 'darkgoldenrod3', 'darkolivegreen1', 'darkorange1', + 'darkorchid1', 'darkseagreen', 'darkslategray4', + 'deeppink1', 'deepskyblue1', 'dimgrey', 'gold1', + 'goldenrod', 'burlywood2', 'bisque1', 'brown1', + 'chartreuse2', 'cornsilk', 'darksalmon') maxModuleCalls = 1 + colorCtr = 0 for modName, modProperties in modules.items(): lineCtr = 0 + modules[modName]['color'] = moduleColors[colorCtr] + colorCtr += 1 + if colorCtr >= len(moduleColors): + colorCtr = 0 for line in modules[modName]['lines']: if line.strip().startswith('def '): name = line.split('def ')[1].split('(')[0] @@ -2909,7 +2920,9 @@ def testFunctions(): for modName, modProperties in modules.items(): callGraphStr += ' subgraph cluster_' + modName + ' {\n' callGraphStr += ' label = "' + modName + '";\n' - callGraphStr += ' node [style=filled];\n' + callGraphStr += ' node ' + callGraphStr += '[style=filled fillcolor=' + callGraphStr += modProperties['color'] + '];\n' moduleFunctionsStr = '' for name in modProperties['functions']: if name.startswith('test'): @@ -2924,11 +2937,13 @@ def testFunctions(): for name, properties in functionProperties.items(): if not properties['calls']: continue + modColor = modules[properties['module']]['color'] for calledFunc in properties['calls']: if calledFunc.startswith('test'): continue if calledFunc not in excludeFuncs: - callGraphStr += ' "' + name + '" -> "' + calledFunc + '";\n' + callGraphStr += ' "' + name + '" -> "' + calledFunc + \ + '" [color=' + modColor + '];\n' callGraphStr += '\n}\n' with open('epicyon.dot', 'w+') as fp: From 7c2786535e0cf97edfbb68f16331e3cfbb34d43d Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 19:55:51 +0000 Subject: [PATCH 10/31] Call graph colors --- tests.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/tests.py b/tests.py index bc3b0d312..32dd0b01a 100644 --- a/tests.py +++ b/tests.py @@ -2854,6 +2854,7 @@ def testFunctions(): 'goldenrod', 'burlywood2', 'bisque1', 'brown1', 'chartreuse2', 'cornsilk', 'darksalmon') maxModuleCalls = 1 + maxFunctionCalls = 1 colorCtr = 0 for modName, modProperties in modules.items(): lineCtr = 0 @@ -2868,6 +2869,8 @@ def testFunctions(): getFunctionCalls(name, modules[modName]['lines'], lineCtr, functionProperties) functionProperties[name]['calls'] = callsList.copy() + if len(callsList) > maxFunctionCalls: + maxFunctionCalls = len(callsList) # keep track of which module calls which other module for fn in callsList: modCall = functionProperties[fn]['module'] @@ -2889,14 +2892,17 @@ def testFunctions(): for modName, modProperties in modules.items(): if not modProperties.get('calls'): callGraphStr += ' "' + modName + \ - '" [fillcolor = yellow style=filled];\n' + '" [fillcolor=yellow style=filled];\n' continue - if len(modProperties['calls']) < int(maxModuleCalls / 4): + if len(modProperties['calls']) <= int(maxModuleCalls / 8): callGraphStr += ' "' + modName + \ - '" [fillcolor = orange style=filled];\n' + '" [fillcolor=green style=filled];\n' + elif len(modProperties['calls']) < int(maxModuleCalls / 4): + callGraphStr += ' "' + modName + \ + '" [fillcolor=orange style=filled];\n' else: callGraphStr += ' "' + modName + \ - '" [fillcolor = red style=filled];\n' + '" [fillcolor=red style=filled];\n' callGraphStr += '\n' # connections between modules for modName, modProperties in modules.items(): @@ -2920,24 +2926,39 @@ def testFunctions(): for modName, modProperties in modules.items(): callGraphStr += ' subgraph cluster_' + modName + ' {\n' callGraphStr += ' label = "' + modName + '";\n' - callGraphStr += ' node ' - callGraphStr += '[style=filled fillcolor=' - callGraphStr += modProperties['color'] + '];\n' + callGraphStr += ' node [style=filled];\n' moduleFunctionsStr = '' for name in modProperties['functions']: if name.startswith('test'): continue if name not in excludeFuncs: - moduleFunctionsStr += '"' + name + '" ' + if not functionProperties[name]['calls']: + moduleFunctionsStr += \ + ' "' + name + '" [fillcolor=yellow style=filled];\n' + continue + noOfCalls = len(functionProperties[name]['calls']) + if noOfCalls < int(maxFunctionCalls / 4): + moduleFunctionsStr += ' "' + name + \ + '" [fillcolor=orange style=filled];\n' + else: + moduleFunctionsStr += ' "' + name + \ + '" [fillcolor=red style=filled];\n' + if moduleFunctionsStr: - callGraphStr += ' ' + moduleFunctionsStr + ';\n' + callGraphStr += moduleFunctionsStr + '\n' callGraphStr += ' color=blue;\n' callGraphStr += ' }\n\n' for name, properties in functionProperties.items(): if not properties['calls']: continue - modColor = modules[properties['module']]['color'] + noOfCalls = len(properties['calls']) + if noOfCalls <= int(maxFunctionCalls / 8): + modColor = 'blue' + elif noOfCalls < int(maxFunctionCalls / 4): + modColor = 'green' + else: + modColor = 'red' for calledFunc in properties['calls']: if calledFunc.startswith('test'): continue From 0fe9b2adcdaab3b1bc83483a1a77efbc9b874617 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 22:18:19 +0000 Subject: [PATCH 11/31] Tidying --- inbox.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/inbox.py b/inbox.py index 96d78c4a1..c0bd64098 100644 --- a/inbox.py +++ b/inbox.py @@ -730,19 +730,19 @@ def _personReceiveUpdate(baseDir: str, ' ' + str(personJson)) domainFull = getFullDomain(domain, port) updateDomainFull = getFullDomain(updateDomain, updatePort) - actor = updateDomainFull + '/users/' + updateNickname - if actor not in personJson['id']: - actor = updateDomainFull + '/profile/' + updateNickname - if actor not in personJson['id']: - actor = updateDomainFull + '/channel/' + updateNickname - if actor not in personJson['id']: - actor = updateDomainFull + '/accounts/' + updateNickname - if actor not in personJson['id']: - if debug: - print('actor: ' + actor) - print('id: ' + personJson['id']) - print('DEBUG: Actor does not match id') - return False + usersPaths = ('users', 'profile', 'channel', 'accounts') + usersStrFound = False + for usersStr in usersPaths: + actor = updateDomainFull + '/' + usersStr + '/' + updateNickname + if actor in personJson['id']: + usersStrFound = True + break + if not usersStrFound: + if debug: + print('actor: ' + actor) + print('id: ' + personJson['id']) + print('DEBUG: Actor does not match id') + return False if updateDomainFull == domainFull: if debug: print('DEBUG: You can only receive actor updates ' + From e86559775f2cc3fd7d1a837be362d94f6b9e2bbd Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 22:45:12 +0000 Subject: [PATCH 12/31] Remove peertube default --- webapp_media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp_media.py b/webapp_media.py index ba5fa9971..1328962f3 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -80,7 +80,7 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, # French and German language # These have been chosen based on reported numbers of users # and the content of each has not been reviewed, so mileage could vary - peerTubeSites = ('peertube.mastodon.host', 'open.tube', 'share.tube', + peerTubeSites = ('peertube.mastodon.host', 'share.tube', 'tube.tr4sk.me', 'videos.elbinario.net', 'hkvideo.live', 'peertube.snargol.com', 'tube.22decembre.eu', From cae148459970a40cb2c79db5e7940d08e5f94a91 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 22:47:07 +0000 Subject: [PATCH 13/31] Comment --- webapp_media.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webapp_media.py b/webapp_media.py index 1328962f3..5677d5a30 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -80,6 +80,8 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, # French and German language # These have been chosen based on reported numbers of users # and the content of each has not been reviewed, so mileage could vary + # Also see https://peertube_isolation.frama.io/list/ for + # adversarial instances peerTubeSites = ('peertube.mastodon.host', 'share.tube', 'tube.tr4sk.me', 'videos.elbinario.net', 'hkvideo.live', From 0dc869f77d60b712308d4aefe490015e336ef13b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 22:48:19 +0000 Subject: [PATCH 14/31] Remove invidious site --- webapp_media.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webapp_media.py b/webapp_media.py index 5677d5a30..45d093958 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -39,8 +39,7 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, "allowfullscreen>\n\n" return content - invidiousSites = ('https://invidio.us', - 'https://invidious.snopyta.org', + invidiousSites = ('https://invidious.snopyta.org', 'http://c7hqkpkpemu6e7emz5b4vy' + 'z7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion', 'http://axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4' + From d7da2f2e65a52a9930f4a948ced7618ff13635db Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 22:50:21 +0000 Subject: [PATCH 15/31] Add invidious sites --- webapp_media.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webapp_media.py b/webapp_media.py index 45d093958..b7ed45ccb 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -40,6 +40,13 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, return content invidiousSites = ('https://invidious.snopyta.org', + 'https://yewtu.be', + 'https://tube.connect.cafe', + 'https://invidious.kavin.rocks', + 'https://invidiou.site', + 'https://invidious.tube', + 'https://invidious.xyz', + 'https://invidious.zapashcanon.fr', 'http://c7hqkpkpemu6e7emz5b4vy' + 'z7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion', 'http://axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4' + From cf4d9466eb3f237374c85557d8ace734e4049d3e Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Wed, 23 Dec 2020 23:59:49 +0000 Subject: [PATCH 16/31] Start of custom peertube sites --- blog.py | 21 ++++++---- daemon.py | 90 +++++++++++++++++++++++++++++++------------ inbox.py | 16 +++++--- webapp_confirm.py | 4 +- webapp_frontscreen.py | 11 ++++-- webapp_media.py | 64 +++++++++++++++++------------- webapp_moderation.py | 5 ++- webapp_post.py | 17 ++++++-- webapp_profile.py | 12 ++++-- webapp_search.py | 8 +++- webapp_timeline.py | 59 +++++++++++++++++----------- 11 files changed, 205 insertions(+), 102 deletions(-) diff --git a/blog.py b/blog.py index ec92e825a..bd6df769f 100644 --- a/blog.py +++ b/blog.py @@ -158,6 +158,7 @@ def _htmlBlogPostContent(authorized: bool, nickname: str, domain: str, domainFull: str, postJsonObject: {}, handle: str, restrictToDomain: bool, + peertubeInstances: [], blogSeparator='
') -> str: """Returns the content for a single blog post """ @@ -231,7 +232,8 @@ def _htmlBlogPostContent(authorized: bool, if postJsonObject['object'].get('content'): contentStr = addEmbeddedElements(translate, - postJsonObject['object']['content']) + postJsonObject['object']['content'], + peertubeInstances) if postJsonObject['object'].get('tag'): contentStr = replaceEmojiFromTags(contentStr, postJsonObject['object']['tag'], @@ -375,7 +377,8 @@ def _htmlBlogRemoveCwButton(blogStr: str, translate: {}) -> str: def htmlBlogPost(authorized: bool, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, domainFull: str, - postJsonObject: {}) -> str: + postJsonObject: {}, + peertubeInstances: []) -> str: """Returns a html blog post """ blogStr = '' @@ -390,7 +393,8 @@ def htmlBlogPost(authorized: bool, httpPrefix, translate, nickname, domain, domainFull, postJsonObject, - None, False) + None, False, + peertubeInstances) # show rss links blogStr += '

' @@ -417,7 +421,8 @@ def htmlBlogPost(authorized: bool, def htmlBlogPage(authorized: bool, session, baseDir: str, httpPrefix: str, translate: {}, nickname: str, domain: str, port: int, - noOfItems: int, pageNumber: int) -> str: + noOfItems: int, pageNumber: int, + peertubeInstances: []) -> str: """Returns a html blog page containing posts """ if ' ' in nickname or '@' in nickname or \ @@ -477,7 +482,8 @@ def htmlBlogPage(authorized: bool, session, httpPrefix, translate, nickname, domain, domainFull, item, - None, True) + None, True, + peertubeInstances) if len(timelineJson['orderedItems']) >= noOfItems: blogStr += navigateStr @@ -638,7 +644,8 @@ def _singleBlogAccountNickname(baseDir: str) -> str: def htmlBlogView(authorized: bool, session, baseDir: str, httpPrefix: str, translate: {}, domain: str, port: int, - noOfItems: int) -> str: + noOfItems: int, + peertubeInstances: []) -> str: """Show the blog main page """ blogStr = '' @@ -654,7 +661,7 @@ def htmlBlogView(authorized: bool, return htmlBlogPage(authorized, session, baseDir, httpPrefix, translate, nickname, domain, port, - noOfItems, 1) + noOfItems, 1, peertubeInstances) domainFull = getFullDomain(domain, port) diff --git a/daemon.py b/daemon.py index a4c86ab89..528505949 100644 --- a/daemon.py +++ b/daemon.py @@ -2490,7 +2490,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self.server.YTReplacementDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + self.server.peertubeInstances) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -2541,7 +2542,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.personCache, port, self.server.YTReplacementDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + self.server.peertubeInstances) if historyStr: msg = historyStr.encode('utf-8') msglen = len(msg) @@ -2585,7 +2587,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, self.server.YTReplacementDomain, self.server.showPublishedDateOnly, - self.server.defaultTimeline) + self.server.defaultTimeline, + self.server.peertubeInstances) if profileStr: msg = profileStr.encode('utf-8') msglen = len(msg) @@ -5380,7 +5383,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, self.server.projectVersion, self.server.YTReplacementDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + self.server.peertubeInstances) if hashtagStr: msg = hashtagStr.encode('utf-8') msglen = len(msg) @@ -6341,7 +6345,8 @@ class PubServer(BaseHTTPRequestHandler): __version__, self.server.cachedWebfingers, self.server.personCache, callingDomain, self.server.YTReplacementDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + self.server.peertubeInstances) if deleteStr: deleteStrLen = len(deleteStr) self._set_headers('text/html', deleteStrLen, @@ -6527,6 +6532,7 @@ class PubServer(BaseHTTPRequestHandler): personCache = self.server.personCache projectVersion = self.server.projectVersion ytDomain = self.server.YTReplacementDomain + peertubeInstances = self.server.peertubeInstances msg = \ htmlPostReplies(self.server.cssCache, recentPostsCache, @@ -6543,7 +6549,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, projectVersion, ytDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -6611,6 +6618,7 @@ class PubServer(BaseHTTPRequestHandler): personCache = self.server.personCache projectVersion = self.server.projectVersion ytDomain = self.server.YTReplacementDomain + peertubeInstances = self.server.peertubeInstances msg = \ htmlPostReplies(self.server.cssCache, recentPostsCache, @@ -6627,7 +6635,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, projectVersion, ytDomain, - self.server.showPublishedDateOnly) + self.server.showPublishedDateOnly, + peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -6713,6 +6722,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, actorJson['roles'], None, None) msg = msg.encode('utf-8') @@ -6796,6 +6806,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, actorJson['skills'], None, None) msg = msg.encode('utf-8') @@ -6900,6 +6911,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.YTReplacementDomain showPublishedDateOnly = \ self.server.showPublishedDateOnly + peertubeInstances = \ + self.server.peertubeInstances cssCache = self.server.cssCache msg = \ htmlIndividualPost(cssCache, @@ -6919,7 +6932,8 @@ class PubServer(BaseHTTPRequestHandler): projectVersion, likedBy, ytDomain, - showPublishedDateOnly) + showPublishedDateOnly, + peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7015,6 +7029,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.YTReplacementDomain showPublishedDateOnly = \ self.server.showPublishedDateOnly + peertubeInstances = \ + self.server.peertubeInstances msg = \ htmlIndividualPost(self.server.cssCache, recentPostsCache, @@ -7033,7 +7049,8 @@ class PubServer(BaseHTTPRequestHandler): projectVersion, likedBy, ytDomain, - showPublishedDateOnly) + showPublishedDateOnly, + peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7165,7 +7182,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) if GETstartTime: self._benchmarkGETtimings(GETstartTime, GETtimings, 'show status done', @@ -7291,7 +7309,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.iconsAsButtons, self.server.rssIconAtTop, self.server.publishButtonAtTop, - authorized, self.server.themeName) + authorized, self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7410,7 +7429,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.iconsAsButtons, self.server.rssIconAtTop, self.server.publishButtonAtTop, - authorized, self.server.themeName) + authorized, self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7530,7 +7550,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7650,7 +7671,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7779,7 +7801,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7904,7 +7927,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -7990,7 +8014,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.iconsAsButtons, self.server.rssIconAtTop, self.server.publishButtonAtTop, - authorized, self.server.themeName) + authorized, self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8093,7 +8118,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8216,7 +8242,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8331,7 +8358,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8436,7 +8464,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.rssIconAtTop, self.server.publishButtonAtTop, authorized, moderationActionStr, - self.server.themeName) + self.server.themeName, + self.server.peertubeInstances) msg = msg.encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8535,6 +8564,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, shares, pageNumber, sharesPerPage) msg = msg.encode('utf-8') @@ -8630,6 +8660,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, following, pageNumber, followsPerPage).encode('utf-8') @@ -8725,6 +8756,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, followers, pageNumber, followsPerPage).encode('utf-8') @@ -8795,6 +8827,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.newswire, self.server.themeName, self.server.dormantMonths, + self.server.peertubeInstances, None, None).encode('utf-8') msglen = len(msg) self._set_headers('text/html', msglen, @@ -8860,7 +8893,8 @@ class PubServer(BaseHTTPRequestHandler): translate, nickname, domain, port, - maxPostsInBlogsFeed, pageNumber) + maxPostsInBlogsFeed, pageNumber, + self.server.peertubeInstances) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -9774,7 +9808,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate, self.server.domain, self.server.port, - maxPostsInBlogsFeed) + maxPostsInBlogsFeed, + self.server.peertubeInstances) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -9870,7 +9905,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate, nickname, self.server.domain, self.server.domainFull, - postJsonObject) + postJsonObject, + self.server.peertubeInstances) if msg is not None: msg = msg.encode('utf-8') msglen = len(msg) @@ -13599,6 +13635,9 @@ def runDaemon(sendThreadsTimeoutMins: int, httpd.iconsCache = {} httpd.fontsCache = {} + # TODO load peertube instances + httpd.peertubeInstances = [] + createInitialLastSeen(baseDir, httpPrefix) print('Creating inbox queue') @@ -13619,7 +13658,8 @@ def runDaemon(sendThreadsTimeoutMins: int, httpd.showPublishedDateOnly, httpd.allowNewsFollowers, httpd.maxFollowers, - httpd.allowLocalNetworkAccess), daemon=True) + httpd.allowLocalNetworkAccess, + httpd.peertubeInstances), daemon=True) print('Creating scheduled post thread') httpd.thrPostSchedule = \ diff --git a/inbox.py b/inbox.py index c0bd64098..eb04fe079 100644 --- a/inbox.py +++ b/inbox.py @@ -146,7 +146,8 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, nickname: str, domain: str, port: int, postJsonObject: {}, allowDeletion: bool, boxname: str, - showPublishedDateOnly: bool) -> None: + showPublishedDateOnly: bool, + peertubeInstances: []) -> None: """Converts the json post into html and stores it in a cache This enables the post to be quickly displayed later """ @@ -171,6 +172,7 @@ def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int, avatarUrl, True, allowDeletion, httpPrefix, __version__, boxname, None, showPublishedDateOnly, + peertubeInstances, not isDM(postJsonObject), True, True, False, True) @@ -2039,7 +2041,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, maxMentions: int, maxEmoji: int, translate: {}, unitTest: bool, YTReplacementDomain: str, showPublishedDateOnly: bool, - allowLocalNetworkAccess: bool) -> bool: + allowLocalNetworkAccess: bool, + peertubeInstances: []) -> bool: """ Anything which needs to be done after initial checks have passed """ actor = keyId @@ -2346,7 +2349,8 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, postJsonObject, allowDeletion, boxname, - showPublishedDateOnly) + showPublishedDateOnly, + peertubeInstances) if debug: timeDiff = \ str(int((time.time() - htmlCacheStartTime) * @@ -2446,7 +2450,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, YTReplacementDomain: str, showPublishedDateOnly: bool, allowNewsFollowers: bool, - maxFollowers: int, allowLocalNetworkAccess: bool) -> None: + maxFollowers: int, allowLocalNetworkAccess: bool, + peertubeInstances: []) -> None: """Processes received items and moves them to the appropriate directories """ @@ -2863,7 +2868,8 @@ def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int, translate, unitTest, YTReplacementDomain, showPublishedDateOnly, - allowLocalNetworkAccess) + allowLocalNetworkAccess, + peertubeInstances) if debug: pprint(queueJson['post']) diff --git a/webapp_confirm.py b/webapp_confirm.py index a0f87f744..3649c256d 100644 --- a/webapp_confirm.py +++ b/webapp_confirm.py @@ -28,7 +28,8 @@ def htmlConfirmDelete(cssCache: {}, wfRequest: {}, personCache: {}, callingDomain: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Shows a screen asking to confirm the deletion of a post """ if '/statuses/' not in messageId: @@ -66,6 +67,7 @@ def htmlConfirmDelete(cssCache: {}, httpPrefix, projectVersion, 'outbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, False, False, False, False) deletePostStr += '

' deletePostStr += \ diff --git a/webapp_frontscreen.py b/webapp_frontscreen.py index ab667ed80..ea26b4b4e 100644 --- a/webapp_frontscreen.py +++ b/webapp_frontscreen.py @@ -27,7 +27,8 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, session, wfRequest: {}, personCache: {}, projectVersion: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Shows posts on the front screen of a news instance These should only be public blog posts from the features timeline which is the blog timeline of the news actor @@ -65,6 +66,7 @@ def _htmlFrontScreenPosts(recentPostsCache: {}, maxRecentPosts: int, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, False, False, True, False) if postStr: profileStr += postStr + separatorStr @@ -85,7 +87,9 @@ def htmlFrontScreen(rssIconAtTop: bool, session, wfRequest: {}, personCache: {}, YTReplacementDomain: str, showPublishedDateOnly: bool, - newswire: {}, theme: str, extraJson=None, + newswire: {}, theme: str, + peertubeInstances: [], + extraJson=None, pageNumber=None, maxItemsPerPage=None) -> str: """Show the news instance front screen """ @@ -148,7 +152,8 @@ def htmlFrontScreen(rssIconAtTop: bool, session, wfRequest, personCache, projectVersion, YTReplacementDomain, - showPublishedDateOnly) + licenseStr + showPublishedDateOnly, + peertubeInstances) + licenseStr # Footer which is only used for system accounts profileFooterStr = ' \n' diff --git a/webapp_media.py b/webapp_media.py index b7ed45ccb..f60348056 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -8,6 +8,7 @@ __status__ = "Production" def _addEmbeddedVideoFromSites(translate: {}, content: str, + peertubeInstances: [], width=400, height=300) -> str: """Adds embedded videos """ @@ -88,31 +89,38 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, # and the content of each has not been reviewed, so mileage could vary # Also see https://peertube_isolation.frama.io/list/ for # adversarial instances - peerTubeSites = ('peertube.mastodon.host', 'share.tube', - 'tube.tr4sk.me', 'videos.elbinario.net', - 'hkvideo.live', - 'peertube.snargol.com', 'tube.22decembre.eu', - 'tube.fabrigli.fr', 'libretube.net', 'libre.video', - 'peertube.linuxrocks.online', 'spacepub.space', - 'video.ploud.jp', 'video.omniatv.com', - 'peertube.servebeer.com', - 'tube.tchncs.de', 'tubee.fr', 'video.alternanet.fr', - 'devtube.dev-wiki.de', 'video.samedi.pm', - 'video.irem.univ-paris-diderot.fr', - 'peertube.openstreetmap.fr', 'video.antopie.org', - 'scitech.video', 'tube.4aem.com', 'video.ploud.fr', - 'peervideo.net', 'video.valme.io', - 'videos.pair2jeux.tube', - 'vault.mle.party', 'hostyour.tv', - 'diode.zone', 'visionon.tv', - 'artitube.artifaille.fr', 'peertube.fr', - 'peertube.live', 'kolektiva.media', - 'tube.ac-lyon.fr', 'www.yiny.org', 'betamax.video', - 'tube.piweb.be', 'pe.ertu.be', 'peertube.social', - 'videos.lescommuns.org', 'peertube.nogafa.org', - 'skeptikon.fr', 'video.tedomum.net', - 'tube.p2p.legal', 'tilvids.com', - 'sikke.fi', 'exode.me', 'peertube.video') + if peertubeInstances: + peerTubeSites = peertubeInstances + else: + peerTubeSites = ('peertube.mastodon.host', 'share.tube', + 'tube.tr4sk.me', 'videos.elbinario.net', + 'hkvideo.live', + 'peertube.snargol.com', 'tube.22decembre.eu', + 'tube.fabrigli.fr', 'libretube.net', + 'libre.video', + 'peertube.linuxrocks.online', 'spacepub.space', + 'video.ploud.jp', 'video.omniatv.com', + 'peertube.servebeer.com', + 'tube.tchncs.de', 'tubee.fr', + 'video.alternanet.fr', + 'devtube.dev-wiki.de', 'video.samedi.pm', + 'video.irem.univ-paris-diderot.fr', + 'peertube.openstreetmap.fr', 'video.antopie.org', + 'scitech.video', 'tube.4aem.com', + 'video.ploud.fr', + 'peervideo.net', 'video.valme.io', + 'videos.pair2jeux.tube', + 'vault.mle.party', 'hostyour.tv', + 'diode.zone', 'visionon.tv', + 'artitube.artifaille.fr', 'peertube.fr', + 'peertube.live', 'kolektiva.media', + 'tube.ac-lyon.fr', 'www.yiny.org', + 'betamax.video', + 'tube.piweb.be', 'pe.ertu.be', 'peertube.social', + 'videos.lescommuns.org', 'peertube.nogafa.org', + 'skeptikon.fr', 'video.tedomum.net', + 'tube.p2p.legal', 'tilvids.com', + 'sikke.fi', 'exode.me', 'peertube.video') for site in peerTubeSites: if '"https://' + site in content: url = content.split('"https://' + site)[1] @@ -224,9 +232,11 @@ def _addEmbeddedVideo(translate: {}, content: str, return content -def addEmbeddedElements(translate: {}, content: str) -> str: +def addEmbeddedElements(translate: {}, content: str, + peertubeInstances: []) -> str: """Adds embedded elements for various media types """ - content = _addEmbeddedVideoFromSites(translate, content) + content = _addEmbeddedVideoFromSites(translate, content, + peertubeInstances) content = _addEmbeddedAudio(translate, content) return _addEmbeddedVideo(translate, content) diff --git a/webapp_moderation.py b/webapp_moderation.py index fabb0b895..c18e5163d 100644 --- a/webapp_moderation.py +++ b/webapp_moderation.py @@ -37,7 +37,7 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, rssIconAtTop: bool, publishButtonAtTop: bool, authorized: bool, moderationActionStr: str, - theme: str) -> str: + theme: str, peertubeInstances: []) -> str: """Show the moderation feed as html This is what you see when selecting the "mod" timeline """ @@ -51,7 +51,8 @@ def htmlModeration(cssCache: {}, defaultTimeline: str, newswire, False, False, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, moderationActionStr, theme) + authorized, moderationActionStr, theme, + peertubeInstances) def htmlAccountInfo(cssCache: {}, translate: {}, diff --git a/webapp_post.py b/webapp_post.py index 274a7840c..2daff3307 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -1076,6 +1076,7 @@ def individualPostAsHtml(allowDownloads: bool, httpPrefix: str, projectVersion: str, boxName: str, YTReplacementDomain: str, showPublishedDateOnly: bool, + peertubeInstances: [], showRepeats=True, showIcons=False, manuallyApprovesFollowers=False, @@ -1483,7 +1484,8 @@ def individualPostAsHtml(allowDownloads: bool, if not postIsSensitive: contentStr = objectContent + attachmentStr - contentStr = addEmbeddedElements(translate, contentStr) + contentStr = addEmbeddedElements(translate, contentStr, + peertubeInstances) contentStr = insertQuestion(baseDir, translate, nickname, domain, port, contentStr, postJsonObject, @@ -1499,7 +1501,8 @@ def individualPostAsHtml(allowDownloads: bool, # get the content warning text cwContentStr = objectContent + attachmentStr if not isPatch: - cwContentStr = addEmbeddedElements(translate, cwContentStr) + cwContentStr = addEmbeddedElements(translate, cwContentStr, + peertubeInstances) cwContentStr = \ insertQuestion(baseDir, translate, nickname, domain, port, cwContentStr, postJsonObject, pageNumber) @@ -1571,7 +1574,8 @@ def htmlIndividualPost(cssCache: {}, postJsonObject: {}, httpPrefix: str, projectVersion: str, likedBy: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Show an individual post as html """ postStr = '' @@ -1611,6 +1615,7 @@ def htmlIndividualPost(cssCache: {}, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, authorized, False, False, False) messageId = removeIdEnding(postJsonObject['id']) @@ -1636,6 +1641,7 @@ def htmlIndividualPost(cssCache: {}, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, authorized, False, False, False) + postStr @@ -1664,6 +1670,7 @@ def htmlIndividualPost(cssCache: {}, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, authorized, False, False, False) cssFilename = baseDir + '/epicyon-profile.css' @@ -1680,7 +1687,8 @@ def htmlPostReplies(cssCache: {}, nickname: str, domain: str, port: int, repliesJson: {}, httpPrefix: str, projectVersion: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Show the replies to an individual post as html """ repliesStr = '' @@ -1696,6 +1704,7 @@ def htmlPostReplies(cssCache: {}, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, False, False, False, False) cssFilename = baseDir + '/epicyon-profile.css' diff --git a/webapp_profile.py b/webapp_profile.py index e89c4de32..3f3ed27d8 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -59,7 +59,8 @@ def htmlProfileAfterSearch(cssCache: {}, debug: bool, projectVersion: str, YTReplacementDomain: str, showPublishedDateOnly: bool, - defaultTimeline: str) -> str: + defaultTimeline: str, + peertubeInstances: []) -> str: """Show a profile page after a search for a fediverse address """ if hasUsersPath(profileHandle) or '/@' in profileHandle: @@ -276,6 +277,7 @@ def htmlProfileAfterSearch(cssCache: {}, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, False, False, False, False) i += 1 if i >= 20: @@ -369,6 +371,7 @@ def htmlProfile(rssIconAtTop: bool, YTReplacementDomain: str, showPublishedDateOnly: bool, newswire: {}, theme: str, dormantMonths: int, + peertubeInstances: [], extraJson=None, pageNumber=None, maxItemsPerPage=None) -> str: """Show the profile page as html @@ -625,7 +628,8 @@ def htmlProfile(rssIconAtTop: bool, session, wfRequest, personCache, projectVersion, YTReplacementDomain, - showPublishedDateOnly) + licenseStr + showPublishedDateOnly, + peertubeInstances) + licenseStr elif selected == 'following': profileStr += \ _htmlProfileFollowing(translate, baseDir, httpPrefix, @@ -671,7 +675,8 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, session, wfRequest: {}, personCache: {}, projectVersion: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Shows posts on the profile screen These should only be public posts """ @@ -709,6 +714,7 @@ def _htmlProfilePosts(recentPostsCache: {}, maxRecentPosts: int, httpPrefix, projectVersion, 'inbox', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, False, False, False, True, False) if postStr: profileStr += postStr + separatorStr diff --git a/webapp_search.py b/webapp_search.py index eab07e191..13b99a6f1 100644 --- a/webapp_search.py +++ b/webapp_search.py @@ -506,7 +506,8 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, personCache: {}, port: int, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Show a page containing search results for your post history """ if historysearch.startswith('!'): @@ -579,6 +580,7 @@ def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str, 'search', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, showIndividualPostIcons, showIndividualPostIcons, False, False, False) @@ -599,7 +601,8 @@ def htmlHashtagSearch(cssCache: {}, session, wfRequest: {}, personCache: {}, httpPrefix: str, projectVersion: str, YTReplacementDomain: str, - showPublishedDateOnly: bool) -> str: + showPublishedDateOnly: bool, + peertubeInstances: []) -> str: """Show a page containing search results for a hashtag """ if hashtag.startswith('#'): @@ -745,6 +748,7 @@ def htmlHashtagSearch(cssCache: {}, 'search', YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, showRepeats, showIcons, manuallyApprovesFollowers, showPublicOnly, diff --git a/webapp_timeline.py b/webapp_timeline.py index 0cb01c850..4aeae76c1 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -61,7 +61,8 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, publishButtonAtTop: bool, authorized: bool, moderationActionStr: str, - theme: str) -> str: + theme: str, + peertubeInstances: []) -> str: """Show the timeline as html """ enableTimingLog = False @@ -566,6 +567,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, boxName, YTReplacementDomain, showPublishedDateOnly, + peertubeInstances, boxName != 'dm', showIndividualPostIcons, manuallyApproveFollowers, @@ -720,7 +722,8 @@ def htmlShares(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the shares timeline as html """ manuallyApproveFollowers = \ @@ -739,7 +742,7 @@ def htmlShares(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInbox(cssCache: {}, defaultTimeline: str, @@ -757,7 +760,8 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the inbox as html """ manuallyApproveFollowers = \ @@ -776,7 +780,7 @@ def htmlInbox(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlBookmarks(cssCache: {}, defaultTimeline: str, @@ -794,7 +798,8 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the bookmarks as html """ manuallyApproveFollowers = \ @@ -813,7 +818,7 @@ def htmlBookmarks(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlEvents(cssCache: {}, defaultTimeline: str, @@ -831,7 +836,8 @@ def htmlEvents(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the events as html """ manuallyApproveFollowers = \ @@ -850,7 +856,7 @@ def htmlEvents(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxDMs(cssCache: {}, defaultTimeline: str, @@ -868,7 +874,8 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the DM timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -882,7 +889,7 @@ def htmlInboxDMs(cssCache: {}, defaultTimeline: str, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxReplies(cssCache: {}, defaultTimeline: str, @@ -900,7 +907,8 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the replies timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -915,7 +923,7 @@ def htmlInboxReplies(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxMedia(cssCache: {}, defaultTimeline: str, @@ -933,7 +941,8 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the media timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -948,7 +957,7 @@ def htmlInboxMedia(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, @@ -966,7 +975,8 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the blogs timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -981,7 +991,7 @@ def htmlInboxBlogs(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, @@ -1000,7 +1010,8 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, rssIconAtTop: bool, publishButtonAtTop: bool, authorized: bool, - theme: str) -> str: + theme: str, + peertubeInstances: []) -> str: """Show the features timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1015,7 +1026,7 @@ def htmlInboxFeatures(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlInboxNews(cssCache: {}, defaultTimeline: str, @@ -1033,7 +1044,8 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the news timeline as html """ return htmlTimeline(cssCache, defaultTimeline, @@ -1048,7 +1060,7 @@ def htmlInboxNews(cssCache: {}, defaultTimeline: str, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) def htmlOutbox(cssCache: {}, defaultTimeline: str, @@ -1066,7 +1078,8 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, iconsAsButtons: bool, rssIconAtTop: bool, publishButtonAtTop: bool, - authorized: bool, theme: str) -> str: + authorized: bool, theme: str, + peertubeInstances: []) -> str: """Show the Outbox as html """ manuallyApproveFollowers = \ @@ -1082,4 +1095,4 @@ def htmlOutbox(cssCache: {}, defaultTimeline: str, newswire, False, False, positiveVoting, showPublishAsIcon, fullWidthTimelineButtonHeader, iconsAsButtons, rssIconAtTop, publishButtonAtTop, - authorized, None, theme) + authorized, None, theme, peertubeInstances) From dadf850e5b12530feb7c242f802362e84d4e099b Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 09:45:41 +0000 Subject: [PATCH 17/31] Load peertube instances from file --- daemon.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 528505949..559358eaa 100644 --- a/daemon.py +++ b/daemon.py @@ -13635,8 +13635,15 @@ def runDaemon(sendThreadsTimeoutMins: int, httpd.iconsCache = {} httpd.fontsCache = {} - # TODO load peertube instances + # load peertube instances from file into a list httpd.peertubeInstances = [] + peertubeInstancesFilename = baseDir + '/accounts/peertube.txt' + if os.path.isfile(peertubeInstancesFilename): + with open(peertubeInstancesFilename, 'r') as fp: + peertubeStr = fp.read() + if peertubeStr: + peertubeStr = peertubeStr.replace('\r', '') + httpd.peertubeInstances = peertubeStr.split('\n') createInitialLastSeen(baseDir, httpPrefix) From f40739e9866e4286aa751a975fa4531ba6cfab4a Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 10:13:21 +0000 Subject: [PATCH 18/31] Comments --- webapp_media.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/webapp_media.py b/webapp_media.py index f60348056..052061eaf 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -83,15 +83,17 @@ def _addEmbeddedVideoFromSites(translate: {}, content: str, return content if '"https://' in content: - # A selection of the current larger peertube sites, mostly - # French and German language - # These have been chosen based on reported numbers of users - # and the content of each has not been reviewed, so mileage could vary - # Also see https://peertube_isolation.frama.io/list/ for - # adversarial instances if peertubeInstances: peerTubeSites = peertubeInstances else: + # A default selection of the current larger peertube sites, + # mostly French and German language + # These have been chosen based on reported numbers of users + # and the content of each has not been reviewed, so mileage + # could vary + # Also see https://peertube_isolation.frama.io/list/ for + # adversarial instances. Nothing in that list should be + # in the defaults below. peerTubeSites = ('peertube.mastodon.host', 'share.tube', 'tube.tr4sk.me', 'videos.elbinario.net', 'hkvideo.live', From 670bbd298418867a1aae7ddb7e340dd2b7f84c13 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 10:18:34 +0000 Subject: [PATCH 19/31] Tidying --- daemon.py | 9 ++------- webapp_media.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/daemon.py b/daemon.py index 559358eaa..35d595654 100644 --- a/daemon.py +++ b/daemon.py @@ -136,6 +136,7 @@ from webapp_timeline import htmlInboxBlogs from webapp_timeline import htmlInboxNews from webapp_timeline import htmlInboxFeatures from webapp_timeline import htmlOutbox +from webapp_media import loadPeertubeInstances from webapp_moderation import htmlAccountInfo from webapp_moderation import htmlModeration from webapp_moderation import htmlModerationInfo @@ -13637,13 +13638,7 @@ def runDaemon(sendThreadsTimeoutMins: int, # load peertube instances from file into a list httpd.peertubeInstances = [] - peertubeInstancesFilename = baseDir + '/accounts/peertube.txt' - if os.path.isfile(peertubeInstancesFilename): - with open(peertubeInstancesFilename, 'r') as fp: - peertubeStr = fp.read() - if peertubeStr: - peertubeStr = peertubeStr.replace('\r', '') - httpd.peertubeInstances = peertubeStr.split('\n') + loadPeertubeInstances(baseDir, httpd.peertubeInstances) createInitialLastSeen(baseDir, httpPrefix) diff --git a/webapp_media.py b/webapp_media.py index 052061eaf..dac29749d 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -6,6 +6,26 @@ __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" +import os + + +def loadPeertubeInstances(baseDir: str, peertubeInstances: []) -> None: + """Loads peertube instances from file into the given list + """ + peertubeList = None + peertubeInstancesFilename = baseDir + '/accounts/peertube.txt' + if os.path.isfile(peertubeInstancesFilename): + with open(peertubeInstancesFilename, 'r') as fp: + peertubeStr = fp.read() + if peertubeStr: + peertubeStr = peertubeStr.replace('\r', '') + peertubeList = peertubeStr.split('\n') + if peertubeList: + for url in peertubeList: + if url in peertubeInstances: + continue + peertubeInstances.append(url) + def _addEmbeddedVideoFromSites(translate: {}, content: str, peertubeInstances: [], From 3d16d93d869da20a18c13960c5cddeddcad019ee Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 11:42:23 +0000 Subject: [PATCH 20/31] Show list of peertube instances on admin profile --- daemon.py | 25 ++++++++++++++++++++++++- translations/ar.json | 4 +++- translations/ca.json | 4 +++- translations/cy.json | 4 +++- translations/de.json | 4 +++- translations/en.json | 4 +++- translations/es.json | 4 +++- translations/fr.json | 4 +++- translations/ga.json | 4 +++- translations/hi.json | 4 +++- translations/it.json | 4 +++- translations/ja.json | 4 +++- translations/oc.json | 4 +++- translations/pt.json | 4 +++- translations/ru.json | 4 +++- translations/zh.json | 4 +++- webapp_media.py | 23 ++++++++++++++++++----- webapp_profile.py | 17 ++++++++++++++++- 18 files changed, 103 insertions(+), 22 deletions(-) diff --git a/daemon.py b/daemon.py index 35d595654..17572f92f 100644 --- a/daemon.py +++ b/daemon.py @@ -4565,6 +4565,27 @@ class PubServer(BaseHTTPRequestHandler): if os.path.isfile(allowedInstancesFilename): os.remove(allowedInstancesFilename) + # save peertube instances list + peertubeInstancesFilename = \ + baseDir + '/accounts/peertube.txt' + if fields.get('ptInstances'): + self.server.peertubeInstances.clear() + with open(peertubeInstancesFilename, 'w+') as aFile: + aFile.write(fields['ptInstances']) + ptInstancesList = fields['ptInstances'].split('\n') + if ptInstancesList: + for url in ptInstancesList: + url = url.strip() + if not url: + continue + if url in self.server.peertubeInstances: + continue + self.server.peertubeInstances.append(url) + else: + if os.path.isfile(peertubeInstancesFilename): + os.remove(peertubeInstancesFilename) + self.server.peertubeInstances.clear() + # save git project names list gitProjectsFilename = \ baseDir + '/accounts/' + \ @@ -9381,6 +9402,7 @@ class PubServer(BaseHTTPRequestHandler): """Show the edit profile screen """ if '/users/' in path and path.endswith('/editprofile'): + peertubeInstances = self.server.peertubeInstances msg = htmlEditProfile(self.server.cssCache, translate, baseDir, @@ -9388,7 +9410,8 @@ class PubServer(BaseHTTPRequestHandler): port, httpPrefix, self.server.defaultTimeline, - self.server.themeName).encode('utf-8') + self.server.themeName, + peertubeInstances).encode('utf-8') if msg: msglen = len(msg) self._set_headers('text/html', msglen, diff --git a/translations/ar.json b/translations/ar.json index 630067626..c0288c1e5 100644 --- a/translations/ar.json +++ b/translations/ar.json @@ -347,5 +347,7 @@ "Filter out words": "تصفية الكلمات", "Unfilter": "غير مرشح", "Unfilter words": "الكلمات غير المصفاة", - "Show Accounts": "إظهار الحسابات" + "Show Accounts": "إظهار الحسابات", + "Peertube Instances": "مثيلات Peertube", + "Show video previews for the following Peertube sites.": "إظهار معاينات الفيديو لمواقع Peertube التالية." } diff --git a/translations/ca.json b/translations/ca.json index 853f94914..ffc4725d9 100644 --- a/translations/ca.json +++ b/translations/ca.json @@ -347,5 +347,7 @@ "Filter out words": "Filtra les paraules", "Unfilter": "Sense filtre", "Unfilter words": "Paraules sense filtre", - "Show Accounts": "Mostra comptes" + "Show Accounts": "Mostra comptes", + "Peertube Instances": "Instàncies de Peertube", + "Show video previews for the following Peertube sites.": "Mostra les previsualitzacions de vídeo dels següents llocs de Peertube." } diff --git a/translations/cy.json b/translations/cy.json index eafc25d65..cd659af7a 100644 --- a/translations/cy.json +++ b/translations/cy.json @@ -347,5 +347,7 @@ "Filter out words": "Hidlo geiriau", "Unfilter": "Di-hid", "Unfilter words": "Geiriau di-hid", - "Show Accounts": "Dangos Cyfrifon" + "Show Accounts": "Dangos Cyfrifon", + "Peertube Instances": "Camau Peertube", + "Show video previews for the following Peertube sites.": "Dangos rhagolygon fideo ar gyfer y safleoedd Peertube canlynol." } diff --git a/translations/de.json b/translations/de.json index 6749671f9..e0e757b7d 100644 --- a/translations/de.json +++ b/translations/de.json @@ -347,5 +347,7 @@ "Filter out words": "Wörter herausfiltern", "Unfilter": "Filter entfernen", "Unfilter words": "Wörter herausfiltern", - "Show Accounts": "Konten anzeigen" + "Show Accounts": "Konten anzeigen", + "Peertube Instances": "Peertube-Instanzen", + "Show video previews for the following Peertube sites.": "Zeigen Sie eine Videovorschau für die folgenden Peertube-Websites an." } diff --git a/translations/en.json b/translations/en.json index 6f3c33440..6ea42b2c6 100644 --- a/translations/en.json +++ b/translations/en.json @@ -347,5 +347,7 @@ "Filter out words": "Filter out words", "Unfilter": "Unfilter", "Unfilter words": "Unfilter words", - "Show Accounts": "Show Accounts" + "Show Accounts": "Show Accounts", + "Peertube Instances": "Peertube Instances", + "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites." } diff --git a/translations/es.json b/translations/es.json index dcf167a97..49293f2e5 100644 --- a/translations/es.json +++ b/translations/es.json @@ -347,5 +347,7 @@ "Filter out words": "Filtrar palabras", "Unfilter": "Unfilter", "Unfilter words": "Palabras sin filtrar", - "Show Accounts": "Mostrar cuentas" + "Show Accounts": "Mostrar cuentas", + "Peertube Instances": "Instancias de Peertube", + "Show video previews for the following Peertube sites.": "Muestre vistas previas de video para los siguientes sitios de Peertube." } diff --git a/translations/fr.json b/translations/fr.json index 3d8dc2b55..ede3e55fa 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -347,5 +347,7 @@ "Filter out words": "Filtrer les mots", "Unfilter": "Non filtrer", "Unfilter words": "Mots non filtrés", - "Show Accounts": "Afficher les comptes" + "Show Accounts": "Afficher les comptes", + "Peertube Instances": "Instances Peertube", + "Show video previews for the following Peertube sites.": "Afficher des aperçus vidéo pour les sites Peertube suivants." } diff --git a/translations/ga.json b/translations/ga.json index a4e46b1f0..af58fe18d 100644 --- a/translations/ga.json +++ b/translations/ga.json @@ -347,5 +347,7 @@ "Filter out words": "Scag focail amach", "Unfilter": "Neamhleithleach", "Unfilter words": "Focail neamhleithleacha", - "Show Accounts": "Taispeáin Cuntais" + "Show Accounts": "Taispeáin Cuntais", + "Peertube Instances": "Imeachtaí Peertube", + "Show video previews for the following Peertube sites.": "Taispeáin réamhamharcanna físe do na suíomhanna Peertube seo a leanas." } diff --git a/translations/hi.json b/translations/hi.json index 49f6d60e2..9c0f78c50 100644 --- a/translations/hi.json +++ b/translations/hi.json @@ -347,5 +347,7 @@ "Filter out words": "शब्दों को फ़िल्टर करें", "Unfilter": "Unfilter", "Unfilter words": "अनफ़िल्टर शब्द", - "Show Accounts": "खाते दिखाएं" + "Show Accounts": "खाते दिखाएं", + "Peertube Instances": "Peertube उदाहरण", + "Show video previews for the following Peertube sites.": "निम्नलिखित Peertube साइटों के लिए वीडियो पूर्वावलोकन दिखाएं।" } diff --git a/translations/it.json b/translations/it.json index bd4b3db9c..36b2fdc10 100644 --- a/translations/it.json +++ b/translations/it.json @@ -347,5 +347,7 @@ "Filter out words": "Filtra le parole", "Unfilter": "Unfilter", "Unfilter words": "Parole non filtrate", - "Show Accounts": "Mostra account" + "Show Accounts": "Mostra account", + "Peertube Instances": "Istanze di Peertube", + "Show video previews for the following Peertube sites.": "Mostra le anteprime dei video per i seguenti siti Peertube." } diff --git a/translations/ja.json b/translations/ja.json index e2029f181..079f7bd07 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -347,5 +347,7 @@ "Filter out words": "単語を除外する", "Unfilter": "フィルタリング解除", "Unfilter words": "単語のフィルタリングを解除する", - "Show Accounts": "アカウントを表示する" + "Show Accounts": "アカウントを表示する", + "Peertube Instances": "Peertubeインスタンス", + "Show video previews for the following Peertube sites.": "次のPeertubeサイトのビデオプレビューを表示します。" } diff --git a/translations/oc.json b/translations/oc.json index 0468223b1..2b7e26f7e 100644 --- a/translations/oc.json +++ b/translations/oc.json @@ -343,5 +343,7 @@ "Filter out words": "Filter out words", "Unfilter": "Unfilter", "Unfilter words": "Unfilter words", - "Show Accounts": "Show Accounts" + "Show Accounts": "Show Accounts", + "Peertube Instances": "Peertube Instances", + "Show video previews for the following Peertube sites.": "Show video previews for the following Peertube sites." } diff --git a/translations/pt.json b/translations/pt.json index b4cb1996a..67b1f3891 100644 --- a/translations/pt.json +++ b/translations/pt.json @@ -347,5 +347,7 @@ "Filter out words": "Filtrar palavras", "Unfilter": "Unfilter", "Unfilter words": "Palavras sem filtro", - "Show Accounts": "Mostrar contas" + "Show Accounts": "Mostrar contas", + "Peertube Instances": "Instâncias Peertube", + "Show video previews for the following Peertube sites.": "Mostrar visualizações de vídeo para os seguintes sites Peertube." } diff --git a/translations/ru.json b/translations/ru.json index a95083395..3236b46fa 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -347,5 +347,7 @@ "Filter out words": "Отфильтровать слова", "Unfilter": "Нефильтровать", "Unfilter words": "Не фильтровать слова", - "Show Accounts": "Показать счета" + "Show Accounts": "Показать счета", + "Peertube Instances": "Экземпляры Peertube", + "Show video previews for the following Peertube sites.": "Показать превью видео для следующих сайтов Peertube." } diff --git a/translations/zh.json b/translations/zh.json index 36ea2b053..fe7a8e92e 100644 --- a/translations/zh.json +++ b/translations/zh.json @@ -347,5 +347,7 @@ "Filter out words": "过滤掉单词", "Unfilter": "取消过滤", "Unfilter words": "未过滤字词", - "Show Accounts": "显示帐户" + "Show Accounts": "显示帐户", + "Peertube Instances": "Peertube实例", + "Show video previews for the following Peertube sites.": "显示以下Peertube网站的视频预览。" } diff --git a/webapp_media.py b/webapp_media.py index dac29749d..73ba89ec8 100644 --- a/webapp_media.py +++ b/webapp_media.py @@ -20,11 +20,24 @@ def loadPeertubeInstances(baseDir: str, peertubeInstances: []) -> None: if peertubeStr: peertubeStr = peertubeStr.replace('\r', '') peertubeList = peertubeStr.split('\n') - if peertubeList: - for url in peertubeList: - if url in peertubeInstances: - continue - peertubeInstances.append(url) + if not peertubeList: + return + for url in peertubeList: + if url in peertubeInstances: + continue + peertubeInstances.append(url) + + +def savePeertubeInstances(baseDir: str, peertubeInstances: []) -> None: + """Saves peertube instances to file from the given list + """ + peertubeStr = '' + for url in peertubeInstances: + peertubeStr += url.strip() + '\n' + + peertubeInstancesFilename = baseDir + '/accounts/peertube.txt' + with open(peertubeInstancesFilename, 'w+') as fp: + fp.write(peertubeStr) def _addEmbeddedVideoFromSites(translate: {}, content: str, diff --git a/webapp_profile.py b/webapp_profile.py index 3f3ed27d8..74f80e425 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -836,7 +836,8 @@ def _htmlProfileShares(actor: str, translate: {}, def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, domain: str, port: int, httpPrefix: str, - defaultTimeline: str, theme: str) -> str: + defaultTimeline: str, theme: str, + peertubeInstances: []) -> str: """Shows the edit profile screen """ imageFormats = getImageFormats() @@ -1427,6 +1428,20 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, ' \n' + editProfileForm += \ + '
\n' + idx = 'Show video previews for the following Peertube sites.' + editProfileForm += \ + '
\n' + peertubeInstancesStr = '' + for url in peertubeInstances: + peertubeInstancesStr += url + '\n' + editProfileForm += \ + ' \n' + editProfileForm += ' \n' editProfileForm += '
\n' editProfileForm += \ From f16ae830b0ee11cf3240f8e02914de58b93fbf55 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 11:49:32 +0000 Subject: [PATCH 21/31] Check admin status when saving peertube sites list --- daemon.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/daemon.py b/daemon.py index 17572f92f..0899d2cbb 100644 --- a/daemon.py +++ b/daemon.py @@ -4566,24 +4566,30 @@ class PubServer(BaseHTTPRequestHandler): os.remove(allowedInstancesFilename) # save peertube instances list - peertubeInstancesFilename = \ + peertubeInstancesFile = \ baseDir + '/accounts/peertube.txt' if fields.get('ptInstances'): - self.server.peertubeInstances.clear() - with open(peertubeInstancesFilename, 'w+') as aFile: - aFile.write(fields['ptInstances']) - ptInstancesList = fields['ptInstances'].split('\n') - if ptInstancesList: - for url in ptInstancesList: - url = url.strip() - if not url: - continue - if url in self.server.peertubeInstances: - continue - self.server.peertubeInstances.append(url) + adminNickname = \ + getConfigParam(baseDir, 'admin') + if adminNickname and \ + path.startswith('/users/' + + adminNickname + '/'): + self.server.peertubeInstances.clear() + with open(peertubeInstancesFile, 'w+') as aFile: + aFile.write(fields['ptInstances']) + ptInstancesList = \ + fields['ptInstances'].split('\n') + if ptInstancesList: + for url in ptInstancesList: + url = url.strip() + if not url: + continue + if url in self.server.peertubeInstances: + continue + self.server.peertubeInstances.append(url) else: - if os.path.isfile(peertubeInstancesFilename): - os.remove(peertubeInstancesFilename) + if os.path.isfile(peertubeInstancesFile): + os.remove(peertubeInstancesFile) self.server.peertubeInstances.clear() # save git project names list From 98f5bab920dce1b0545a8720266525a7b80c9543 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Thu, 24 Dec 2020 11:56:17 +0000 Subject: [PATCH 22/31] Move peertube list into admin area --- webapp_profile.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/webapp_profile.py b/webapp_profile.py index 74f80e425..a8926f5b7 100644 --- a/webapp_profile.py +++ b/webapp_profile.py @@ -1033,6 +1033,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, themesDropdown = '' instanceStr = '' editorsStr = '' + peertubeStr = '' adminNickname = getConfigParam(baseDir, 'admin') if adminNickname: @@ -1149,6 +1150,21 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, '
\n' editProfileForm += '
\n' editProfileForm += \ @@ -1454,7 +1456,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str, editProfileForm += ' \n' editProfileForm += skillsStr + themesDropdown - editProfileForm += moderatorsStr + editorsStr + editProfileForm += moderatorsStr + editorsStr + peertubeStr editProfileForm += '
\n' + instanceStr editProfileForm += '
\n' editProfileForm += '