From f609e365da7b020096f5f2a15fe0bd940eef3900 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Tue, 2 Jul 2019 10:25:29 +0100 Subject: [PATCH] Announce function --- README.md | 2 +- announce.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++ daemon.py | 10 +++++++-- epicyon.py | 21 ++++++++--------- posts.py | 44 +++++++++++------------------------- session.py | 6 ++++- tests.py | 16 ++++++------- utils.py | 32 ++++++++++++++++++++++++++ 8 files changed, 143 insertions(+), 53 deletions(-) create mode 100644 announce.py create mode 100644 utils.py diff --git a/README.md b/README.md index 1d9b3e7f..2c4ed77c 100644 --- a/README.md +++ b/README.md @@ -9,5 +9,5 @@ Also: https://raw.githubusercontent.com/w3c/activitypub/gh-pages/activitypub-tut ## Install ``` bash -sudo pacman -S python-pysocks python-pycryptodome python-beautifulsoup4 +sudo pacman -S python-pysocks python-pycryptodome python-beautifulsoup4 python-requests-toolbelt ``` \ No newline at end of file diff --git a/announce.py b/announce.py new file mode 100644 index 00000000..8bf44f60 --- /dev/null +++ b/announce.py @@ -0,0 +1,65 @@ +__filename__ = "announce.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "0.0.1" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import json +import commentjson +from utils import getStatusNumber +from utils import createOutboxDir + +def createAnnounce(baseDir: str,username: str, domain: str, port: int,toUrl: str, ccUrl: str, https: bool, objectUrl: str, saveToFile: bool) -> {}: + """Creates an announce message + Typically toUrl will be https://www.w3.org/ns/activitystreams#Public + and ccUrl might be a specific person favorited or repeated and the followers url + objectUrl is typically the url of the message, corresponding to url or atomUri in createPostBase + """ + prefix='https' + if not https: + prefix='http' + + if port!=80 and port!=443: + domain=domain+':'+str(port) + + statusNumber,published = getStatusNumber() + newAnnounceId=prefix+'://'+domain+'/users/'+username+'/statuses/'+statusNumber + newAnnounce = { + 'actor': prefix+'://'+domain+'/users/'+username, + 'atomUri': prefix+'://'+domain+'/users/'+username+'/statuses/'+statusNumber, + 'cc': [], + 'id': newAnnounceId+'/activity', + 'object': objectUrl, + 'published': published, + 'to': [toUrl], + 'type': 'Announce' + } + if ccUrl: + if len(ccUrl)>0: + newAnnounce['cc']=ccUrl + if saveToFile: + if ':' in domain: + domain=domain.split(':')[0] + outboxDir = createOutboxDir(username,domain,baseDir) + filename=outboxDir+'/'+newAnnounceId.replace('/','#')+'.json' + with open(filename, 'w') as fp: + commentjson.dump(newAnnounce, fp, indent=4, sort_keys=False) + return newAnnounce + +def announcePublic(baseDir: str,username: str, domain: str, port: int, https: bool, objectUrl: str, saveToFile: bool) -> {}: + """Makes a public announcement + """ + prefix='https' + if not https: + prefix='http' + + fromDomain=domain + if port!=80 and port!=443: + fromDomain=fromDomain+':'+str(port) + + toUrl = 'https://www.w3.org/ns/activitystreams#Public' + ccUrl = prefix + '://'+fromDomain+'/users/'+username+'/followers' + return createAnnounce(baseDir,username, domain, port,toUrl, ccUrl, https, objectUrl, saveToFile) + diff --git a/daemon.py b/daemon.py index 950ab1f6..526644f2 100644 --- a/daemon.py +++ b/daemon.py @@ -220,7 +220,7 @@ class PubServer(BaseHTTPRequestHandler): currSessionTime=int(time.time()) if currSessionTime-self.server.sessionLastUpdate>600: self.server.sessionLastUpdate=currSessionTime - self.server.session = createSession(self.server.useTor) + self.server.session = createSession(self.server.domain,self.server.port,self.server.useTor) print('**************** POST get public key of '+personUrl+' from '+self.server.baseDir) pubKey=getPersonPubKey(self.server.session,personUrl,self.server.personCache) if not pubKey: @@ -230,6 +230,12 @@ class PubServer(BaseHTTPRequestHandler): self.server.POSTbusy=False return print('**************** POST check signature') + if not verifyPostHeaders(self.server.https, pubKey, self.headers, '/inbox' ,False, json.dumps(messageJson)): + print('**************** POST signature verification failed') + self.send_response(401) + self.end_headers() + self.server.POSTbusy=False + return print('**************** POST valid') pprint(messageJson) # add a property to the object, just to mess with data @@ -261,7 +267,7 @@ def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None: httpd.personCache={} httpd.cachedWebfingers={} httpd.useTor=useTor - httpd.session = createSession(useTor) + httpd.session = createSession(domain,port,useTor) httpd.sessionLastUpdate=int(time.time()) httpd.lastGET=0 httpd.lastPOST=0 diff --git a/epicyon.py b/epicyon.py index d2b31d00..3feb3249 100644 --- a/epicyon.py +++ b/epicyon.py @@ -10,7 +10,7 @@ from person import createPerson from person import setPreferredUsername from person import setBio from webfinger import webfingerHandle -from posts import getUserPosts +from posts import getPosts from posts import createPublicPost from posts import deleteAllPosts from posts import createOutbox @@ -35,6 +35,7 @@ from follow import unfollowPerson from follow import unfollowerOfPerson from tests import testPostMessageBetweenServers from tests import runAllTests +from announce import announcePublic runAllTests() @@ -46,7 +47,7 @@ port=6227 https=False useTor=False baseDir=os.getcwd() -session = createSession(useTor) +session = createSession(domain,port,useTor) personCache={} cachedWebfingers={} @@ -84,11 +85,11 @@ setBio(baseDir,username,domain,'Some personal info') #outboxJson=createOutbox(baseDir,username,domain,port,https,2,True,None) #pprint(outboxJson) -testPostMessageBetweenServers() +#testPostMessageBetweenServers() #runDaemon(domain,port,https,federationList,useTor) #testHttpsig() -sys.exit() +#sys.exit() #pprint(person) #print('\n') @@ -99,16 +100,16 @@ wfRequest = webfingerHandle(session,handle,True,cachedWebfingers) if not wfRequest: sys.exit() -personJson,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest,personCache) -pprint(personJson) -sys.exit() +personUrl,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest,personCache,'outbox') +#pprint(personUrl) +#sys.exit() wfResult = json.dumps(wfRequest, indent=4, sort_keys=True) -print(str(wfResult)) -sys.exit() +#print(str(wfResult)) +#sys.exit() maxMentions=10 maxEmoji=10 maxAttachments=5 -userPosts = getUserPosts(session,wfRequest,2,maxMentions,maxEmoji,maxAttachments,federationList) +userPosts = getPosts(session,personUrl,10,maxMentions,maxEmoji,maxAttachments,federationList,personCache) #print(str(userPosts)) diff --git a/posts.py b/posts.py index ffcc402b..c18fdec1 100644 --- a/posts.py +++ b/posts.py @@ -26,6 +26,8 @@ from session import getJson from session import postJson from webfinger import webfingerHandle from httpsig import createSignedHeader +from utils import getStatusNumber +from utils import createOutboxDir try: from BeautifulSoup import BeautifulSoup except ImportError: @@ -113,6 +115,7 @@ def getPersonPubKey(session,personUrl: str,personCache: {}) -> str: return None personJson = getPersonFromCache(personUrl,personCache) if not personJson: + print('************Obtaining public key for '+personUrl) personJson = getJson(session,personUrl,asHeader,None) pubKey=None if personJson.get('publicKey'): @@ -122,14 +125,15 @@ def getPersonPubKey(session,personUrl: str,personCache: {}) -> str: storePersonInCache(personUrl,personJson,personCache) return pubKey -def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: [],personCache: {}) -> {}: - userPosts={} - feedUrl,pubKeyId,pubKey,personId = getPersonBox(session,wfRequest,personCache,'outbox') - if not feedUrl: - return userPosts +def getPosts(session,outboxUrl: str,maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: [],personCache: {}) -> {}: + personPosts={} + if not outboxUrl: + return personPosts + asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'} i = 0 - for item in parseUserFeed(session,feedUrl,asHeader): + for item in parseUserFeed(session,outboxUrl,asHeader): + pprint(item) if not item.get('type'): continue if item['type'] != 'Create': @@ -137,7 +141,7 @@ def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: if not item.get('object'): continue published = item['object']['published'] - if not userPosts.get(published): + if not personPosts.get(published): content = item['object']['content'] mentions=[] @@ -195,7 +199,7 @@ def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: if item['object'].get('sensitive'): sensitive = item['object']['sensitive'] - userPosts[published] = { + personPosts[published] = { "sensitive": sensitive, "inreplyto": inReplyTo, "summary": summary, @@ -211,18 +215,7 @@ def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: if i == maxPosts: break - return userPosts - -def createOutboxDir(username: str,domain: str,baseDir: str) -> str: - """Create an outbox for a person and returns the feed filename and directory - """ - handle=username.lower()+'@'+domain.lower() - if not os.path.isdir(baseDir+'/accounts/'+handle): - os.mkdir(baseDir+'/accounts/'+handle) - outboxDir=baseDir+'/accounts/'+handle+'/outbox' - if not os.path.isdir(outboxDir): - os.mkdir(outboxDir) - return outboxDir + return personPosts def createOutboxArchive(username: str,domain: str,baseDir: str) -> str: """Creates an archive directory for outbox posts @@ -247,17 +240,6 @@ def deleteAllPosts(username: str, domain: str,baseDir: str) -> None: elif os.path.isdir(filePath): shutil.rmtree(filePath) except Exception as e: print(e) - -def getStatusNumber() -> (str,str): - """Returns the status number and published date - """ - currTime=datetime.datetime.utcnow() - daysSinceEpoch=(currTime - datetime.datetime(1970,1,1)).days - # status is the number of seconds since epoch - statusNumber=str(((daysSinceEpoch*24*60*60) + (currTime.hour*60*60) + (currTime.minute*60) + currTime.second)*1000000 + currTime.microsecond) - published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ") - conversationDate=currTime.strftime("%Y-%m-%d") - return statusNumber,published def createPostBase(baseDir: str,username: str, domain: str, port: int,toUrl: str, ccUrl: str, https: bool, content: str, followersOnly: bool, saveToFile: bool, inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}: """Creates a message diff --git a/session.py b/session.py index 10371ff3..c4073e02 100644 --- a/session.py +++ b/session.py @@ -7,12 +7,16 @@ __email__ = "bob@freedombone.net" __status__ = "Production" import requests +from requests_toolbelt.adapters.source import SourceAddressAdapter import json baseDirectory=None -def createSession(onionRoute: bool): +def createSession(domain: str, port: int, onionRoute: bool): session = requests.session() + if domain.startswith('127.') or domain.startswith('192.') or domain.startswith('10.'): + session.mount('http://', SourceAddressAdapter(domain)) + #session.mount('http://', SourceAddressAdapter((domain, port))) if onionRoute: session.proxies = {} session.proxies['http'] = 'socks5h://localhost:9050' diff --git a/tests.py b/tests.py index 131eebfa..252bdf1e 100644 --- a/tests.py +++ b/tests.py @@ -109,8 +109,8 @@ def createServerAlice(path: str,domain: str,port: int,federationList: []): useTor=False privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,username,domain,port,https,True) deleteAllPosts(username,domain,path) - followPerson(path,username,domain,'bob','127.0.10.2:61936',federationList) - followerOfPerson(path,username,domain,'bob','127.0.10.2:61936',federationList) + followPerson(path,username,domain,'bob','127.0.0.100:61936',federationList) + followerOfPerson(path,username,domain,'bob','127.0.0.100:61936',federationList) createPublicPost(path,username, domain, port,https, "No wise fish would go anywhere without a porpoise", False, True) createPublicPost(path,username, domain, port,https, "Curiouser and curiouser!", False, True) createPublicPost(path,username, domain, port,https, "In the gardens of memory, in the palace of dreams, that is where you and I shall meet", False, True) @@ -130,8 +130,8 @@ def createServerBob(path: str,domain: str,port: int,federationList: []): useTor=False privateKeyPem,publicKeyPem,person,wfEndpoint=createPerson(path,username,domain,port,https,True) deleteAllPosts(username,domain,path) - followPerson(path,username,domain,'alice','127.0.10.1:61935',federationList) - followerOfPerson(path,username,domain,'alice','127.0.10.1:61935',federationList) + followPerson(path,username,domain,'alice','127.0.0.50:61935',federationList) + followerOfPerson(path,username,domain,'alice','127.0.0.50:61935',federationList) createPublicPost(path,username, domain, port,https, "It's your life, live it your way.", False, True) createPublicPost(path,username, domain, port,https, "One of the things I've realised is that I am very simple", False, True) createPublicPost(path,username, domain, port,https, "Quantum physics is a bit of a passion of mine", False, True) @@ -150,7 +150,7 @@ def testPostMessageBetweenServers(): https=False useTor=False - federationList=['127.0.0.1','127.0.10.1','127.0.10.2'] + federationList=['127.0.0.50','127.0.0.100'] baseDir=os.getcwd() if not os.path.isdir(baseDir+'/.tests'): @@ -158,12 +158,12 @@ def testPostMessageBetweenServers(): # create the servers aliceDir=baseDir+'/.tests/alice' - aliceDomain='127.0.10.1' + aliceDomain='127.0.0.50' alicePort=61935 thrAlice = threadWithTrace(target=createServerAlice,args=(aliceDir,aliceDomain,alicePort,federationList),daemon=True) bobDir=baseDir+'/.tests/bob' - bobDomain='127.0.10.2' + bobDomain='127.0.0.100' bobPort=61936 thrBob = threadWithTrace(target=createServerBob,args=(bobDir,bobDomain,bobPort,federationList),daemon=True) @@ -180,7 +180,7 @@ def testPostMessageBetweenServers(): print('Alice sends to Bob') os.chdir(aliceDir) - sessionAlice = createSession(useTor) + sessionAlice = createSession(aliceDomain,alicePort,useTor) inReplyTo=None inReplyToAtomUri=None subject=None diff --git a/utils.py b/utils.py new file mode 100644 index 00000000..f2a7b24f --- /dev/null +++ b/utils.py @@ -0,0 +1,32 @@ +__filename__ = "utils.py" +__author__ = "Bob Mottram" +__license__ = "AGPL3+" +__version__ = "0.0.1" +__maintainer__ = "Bob Mottram" +__email__ = "bob@freedombone.net" +__status__ = "Production" + +import os +import datetime + +def getStatusNumber() -> (str,str): + """Returns the status number and published date + """ + currTime=datetime.datetime.utcnow() + daysSinceEpoch=(currTime - datetime.datetime(1970,1,1)).days + # status is the number of seconds since epoch + statusNumber=str(((daysSinceEpoch*24*60*60) + (currTime.hour*60*60) + (currTime.minute*60) + currTime.second)*1000000 + currTime.microsecond) + published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ") + conversationDate=currTime.strftime("%Y-%m-%d") + return statusNumber,published + +def createOutboxDir(username: str,domain: str,baseDir: str) -> str: + """Create an outbox for a person and returns the feed filename and directory + """ + handle=username.lower()+'@'+domain.lower() + if not os.path.isdir(baseDir+'/accounts/'+handle): + os.mkdir(baseDir+'/accounts/'+handle) + outboxDir=baseDir+'/accounts/'+handle+'/outbox' + if not os.path.isdir(outboxDir): + os.mkdir(outboxDir) + return outboxDir