mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fallbacks for GET and POST locking

master
Bob Mottram 2019-07-01 15:30:48 +01:00
parent 220c54ba7d
commit 22090a1aef
6 changed files with 104 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cache.py
View File

@ -8,26 +8,18 @@ __status__ = "Production"
import datetime import datetime
# cache of actor json def storePersonInCache(personUrl: str,personJson: {},personCache: {}) -> None:
# If there are repeated lookups then this helps prevent a lot
# of needless network traffic
personCache = {}
# cached webfinger endpoints
cachedWebfingers = {}
def storePersonInCache(personUrl: str,personJson: {}) -> None:
"""Store an actor in the cache """Store an actor in the cache
""" """
currTime=datetime.datetime.utcnow() currTime=datetime.datetime.utcnow()
personCache[personUrl]={ "actor": personJson, "timestamp": currTime.strftime("%Y-%m-%dT%H:%M:%SZ") } personCache[personUrl]={ "actor": personJson, "timestamp": currTime.strftime("%Y-%m-%dT%H:%M:%SZ") }
def storeWebfingerInCache(handle: str,wf) -> None: def storeWebfingerInCache(handle: str,wf,cachedWebfingers: {}) -> None:
"""Store a webfinger endpoint in the cache """Store a webfinger endpoint in the cache
""" """
cachedWebfingers[handle]=wf cachedWebfingers[handle]=wf
def getPersonFromCache(personUrl: str) -> {}: def getPersonFromCache(personUrl: str,personCache: {}) -> {}:
"""Get an actor from the cache """Get an actor from the cache
""" """
if personCache.get(personUrl): if personCache.get(personUrl):
@ -40,10 +32,9 @@ def getPersonFromCache(personUrl: str) -> {}:
return personCache[personUrl]['actor'] return personCache[personUrl]['actor']
return None return None
def getWebfingerFromCache(handle: str) -> {}: def getWebfingerFromCache(handle: str,cachedWebfingers: {}) -> {}:
"""Get webfinger endpoint from the cache """Get webfinger endpoint from the cache
""" """
if cachedWebfingers.get(handle): if cachedWebfingers.get(handle):
return cachedWebfingers[handle] return cachedWebfingers[handle]
return None return None

91
daemon.py
View File

@ -9,7 +9,7 @@ __status__ = "Production"
from http.server import BaseHTTPRequestHandler, HTTPServer from http.server import BaseHTTPRequestHandler, HTTPServer
#import socketserver #import socketserver
import json import json
import cgi import time
from pprint import pprint from pprint import pprint
from session import createSession from session import createSession
from webfinger import webfingerMeta from webfinger import webfingerMeta
@ -17,6 +17,7 @@ from webfinger import webfingerLookup
from person import personLookup from person import personLookup
from person import personKeyLookup from person import personKeyLookup
from person import personOutboxJson from person import personOutboxJson
from posts import getPersonPubKey
from inbox import inboxPermittedMessage from inbox import inboxPermittedMessage
from follow import getFollowingFeed from follow import getFollowingFeed
import os import os
@ -85,59 +86,59 @@ class PubServer(BaseHTTPRequestHandler):
return True return True
def do_GET(self): def do_GET(self):
try: if self.server.GETbusy:
if self.GETbusy: currTimeGET=int(time.time())
if currTimeGET-self.server.lastGET<10:
self.send_response(429) self.send_response(429)
self.end_headers() self.end_headers()
return return
except: self.server.lastGET=currTimeGET
pass self.server.GETbusy=True
self.GETbusy=True
if not self._permittedDir(self.path): if not self._permittedDir(self.path):
self._404() self._404()
self.GETbusy=False self.server.GETbusy=False
return return
# get webfinger endpoint for a person # get webfinger endpoint for a person
if self._webfinger(): if self._webfinger():
self.GETbusy=False self.server.GETbusy=False
return return
# get outbox feed for a person # get outbox feed for a person
outboxFeed=personOutboxJson(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,maxPostsInFeed) outboxFeed=personOutboxJson(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,maxPostsInFeed)
if outboxFeed: if outboxFeed:
self._set_headers('application/json') self._set_headers('application/json')
self.wfile.write(json.dumps(outboxFeed).encode('utf-8')) self.wfile.write(json.dumps(outboxFeed).encode('utf-8'))
self.GETbusy=False self.server.GETbusy=False
return return
following=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage) following=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage)
if following: if following:
self._set_headers('application/json') self._set_headers('application/json')
self.wfile.write(json.dumps(following).encode('utf-8')) self.wfile.write(json.dumps(following).encode('utf-8'))
self.GETbusy=False self.server.GETbusy=False
return return
followers=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage,'followers') followers=getFollowingFeed(self.server.baseDir,self.server.domain,self.server.port,self.path,self.server.https,followsPerPage,'followers')
if followers: if followers:
self._set_headers('application/json') self._set_headers('application/json')
self.wfile.write(json.dumps(followers).encode('utf-8')) self.wfile.write(json.dumps(followers).encode('utf-8'))
self.GETbusy=False self.server.GETbusy=False
return return
# look up a person # look up a person
getPerson = personLookup(self.server.domain,self.path,self.server.baseDir) getPerson = personLookup(self.server.domain,self.path,self.server.baseDir)
if getPerson: if getPerson:
self._set_headers('application/json') self._set_headers('application/json')
self.wfile.write(json.dumps(getPerson).encode('utf-8')) self.wfile.write(json.dumps(getPerson).encode('utf-8'))
self.GETbusy=False self.server.GETbusy=False
return return
personKey = personKeyLookup(self.server.domain,self.path,self.server.baseDir) personKey = personKeyLookup(self.server.domain,self.path,self.server.baseDir)
if personKey: if personKey:
self._set_headers('text/html; charset=utf-8') self._set_headers('text/html; charset=utf-8')
self.wfile.write(personKey.encode('utf-8')) self.wfile.write(personKey.encode('utf-8'))
self.GETbusy=False self.server.GETbusy=False
return return
# check that a json file was requested # check that a json file was requested
if not self.path.endswith('.json'): if not self.path.endswith('.json'):
self._404() self._404()
self.GETbusy=False self.server.GETbusy=False
return return
# check that the file exists # check that the file exists
filename=self.server.baseDir+self.path filename=self.server.baseDir+self.path
@ -149,27 +150,26 @@ class PubServer(BaseHTTPRequestHandler):
self.wfile.write(json.dumps(contentJson).encode('utf8')) self.wfile.write(json.dumps(contentJson).encode('utf8'))
else: else:
self._404() self._404()
self.GETbusy=False self.server.GETbusy=False
def do_HEAD(self): def do_HEAD(self):
self._set_headers('application/json') self._set_headers('application/json')
def do_POST(self): def do_POST(self):
print('**************** POST recieved!') if self.server.POSTbusy:
try: currTimePOST=int(time.time())
if self.POSTbusy: if currTimePOST-self.server.lastPOST<10:
self.send_response(429) self.send_response(429)
self.end_headers() self.end_headers()
return return
except: self.server.lastPOST=currTimePOST
pass
print('**************** POST ready to receive') print('**************** POST ready to receive')
self.POSTbusy=True self.server.POSTbusy=True
if not self.headers.get('Content-type'): if not self.headers.get('Content-type'):
print('**************** No Content-type') print('**************** No Content-type')
self.send_response(400) self.send_response(400)
self.end_headers() self.end_headers()
self.POSTbusy=False self.server.POSTbusy=False
return return
print('*****************headers: '+str(self.headers)) print('*****************headers: '+str(self.headers))
@ -178,7 +178,7 @@ class PubServer(BaseHTTPRequestHandler):
print("**************** That's no Json!") print("**************** That's no Json!")
self.send_response(400) self.send_response(400)
self.end_headers() self.end_headers()
self.POSTbusy=False self.server.POSTbusy=False
return return
# read the message and convert it into a python dictionary # read the message and convert it into a python dictionary
@ -187,7 +187,7 @@ class PubServer(BaseHTTPRequestHandler):
if length>maxMessageLength: if length>maxMessageLength:
self.send_response(400) self.send_response(400)
self.end_headers() self.end_headers()
self.POSTbusy=False self.server.POSTbusy=False
return return
print('**************** Reading message') print('**************** Reading message')
messageBytes=self.rfile.read(length) messageBytes=self.rfile.read(length)
@ -197,9 +197,34 @@ class PubServer(BaseHTTPRequestHandler):
print('**************** Ah Ah Ah') print('**************** Ah Ah Ah')
self.send_response(403) self.send_response(403)
self.end_headers() self.end_headers()
self.POSTbusy=False self.server.POSTbusy=False
return return
print('**************** POST get handle')
handle=''
print('**************** POST create session')
currSessionTime=int(time.time())
if currSessionTime-self.server.sessionLastUpdate>600:
self.server.sessionLastUpdate=currSessionTime
self.server.session = createSession(self.server.useTor)
print('**************** POST webfinger the handle')
wfRequest = webfingerHandle(self.server.session,handle,self.server.https,self.server.cachedWebfingers)
if not wfRequest:
print('**************** POST unknown webfinger')
self.send_response(401)
self.end_headers()
self.server.POSTbusy=False
return
print('**************** POST get public key')
pubKey=getPersonPubKey(self.server.session,wfRequest,self.server.personCache)
if not pubKey:
print('**************** POST no sender public key')
self.send_response(401)
self.end_headers()
self.server.POSTbusy=False
return
print('**************** POST check signature')
print('**************** POST valid') print('**************** POST valid')
pprint(messageJson) pprint(messageJson)
# add a property to the object, just to mess with data # add a property to the object, just to mess with data
@ -210,7 +235,7 @@ class PubServer(BaseHTTPRequestHandler):
#self.wfile.write(json.dumps(message).encode('utf-8')) #self.wfile.write(json.dumps(message).encode('utf-8'))
self.send_response(200) self.send_response(200)
self.end_headers() self.end_headers()
self.POSTbusy=False self.server.POSTbusy=False
def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None: def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None:
if len(domain)==0: if len(domain)==0:
@ -218,7 +243,6 @@ def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None:
if '.' not in domain: if '.' not in domain:
print('Invalid domain: ' + domain) print('Invalid domain: ' + domain)
return return
session = createSession(useTor)
serverAddress = ('', port) serverAddress = ('', port)
httpd = HTTPServer(serverAddress, PubServer) httpd = HTTPServer(serverAddress, PubServer)
@ -227,5 +251,14 @@ def runDaemon(domain: str,port=80,https=True,fedList=[],useTor=False) -> None:
httpd.https=https httpd.https=https
httpd.federationList=fedList.copy() httpd.federationList=fedList.copy()
httpd.baseDir=os.getcwd() httpd.baseDir=os.getcwd()
httpd.personCache={}
httpd.cachedWebfingers={}
httpd.useTor=useTor
httpd.session = createSession(useTor)
httpd.sessionLastUpdate=int(time.time())
httpd.lastGET=0
httpd.lastPOST=0
httpd.GETbusy=False
httpd.POSTbusy=False
print('Running ActivityPub daemon on ' + domain + ' port ' + str(port)) print('Running ActivityPub daemon on ' + domain + ' port ' + str(port))
httpd.serve_forever() httpd.serve_forever()

7
epicyon.py
View File

@ -47,7 +47,8 @@ https=False
useTor=False useTor=False
baseDir=os.getcwd() baseDir=os.getcwd()
session = createSession(useTor) session = createSession(useTor)
personCache={}
cachedWebfingers={}
clearFollows(baseDir,username,domain) clearFollows(baseDir,username,domain)
followPerson(baseDir,username,domain,'badger','wild.com',federationList) followPerson(baseDir,username,domain,'badger','wild.com',federationList)
@ -94,11 +95,11 @@ sys.exit()
#pprint(wfEndpoint) #pprint(wfEndpoint)
handle="https://mastodon.social/@Gargron" handle="https://mastodon.social/@Gargron"
wfRequest = webfingerHandle(session,handle,True) wfRequest = webfingerHandle(session,handle,True,cachedWebfingers)
if not wfRequest: if not wfRequest:
sys.exit() sys.exit()
personJson,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest) personJson,pubKeyId,pubKey,personId=getPersonBox(session,wfRequest,personCache)
pprint(personJson) pprint(personJson)
sys.exit() sys.exit()

33
posts.py
View File

@ -82,12 +82,12 @@ def parseUserFeed(session,feedUrl: str,asHeader: {}) -> None:
for item in parseUserFeed(session,nextUrl,asHeader): for item in parseUserFeed(session,nextUrl,asHeader):
yield item yield item
def getPersonBox(session,wfRequest,boxName='inbox') -> (str,str,str,str): def getPersonBox(session,wfRequest: {},personCache: {},boxName='inbox') -> (str,str,str,str):
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'} asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
personUrl = getUserUrl(wfRequest) personUrl = getUserUrl(wfRequest)
if not personUrl: if not personUrl:
return None return None
personJson = getPersonFromCache(personUrl) personJson = getPersonFromCache(personUrl,personCache)
if not personJson: if not personJson:
personJson = getJson(session,personUrl,asHeader,None) personJson = getJson(session,personUrl,asHeader,None)
if not personJson.get(boxName): if not personJson.get(boxName):
@ -103,13 +103,30 @@ def getPersonBox(session,wfRequest,boxName='inbox') -> (str,str,str,str):
if personJson['publicKey'].get('publicKeyPem'): if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem'] pubKey=personJson['publicKey']['publicKeyPem']
storePersonInCache(personUrl,personJson) storePersonInCache(personUrl,personJson,personCache)
return personJson[boxName],pubKeyId,pubKey,personId return personJson[boxName],pubKeyId,pubKey,personId
def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: []) -> {}: def getPersonPubKey(session,wfRequest,personCache: {}) -> str:
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
personUrl = getUserUrl(wfRequest)
if not personUrl:
return None
personJson = getPersonFromCache(personUrl,personCache)
if not personJson:
personJson = getJson(session,personUrl,asHeader,None)
pubKey=None
if personJson.get('publicKey'):
if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem']
storePersonInCache(personUrl,personJson,personCache)
return pubKey
def getUserPosts(session,wfRequest: {},maxPosts: int,maxMentions: int,maxEmoji: int,maxAttachments: int,federationList: [],personCache: {}) -> {}:
userPosts={} userPosts={}
feedUrl,pubKeyId,pubKey,personId = getPersonBox(session,wfRequest,'outbox') feedUrl,pubKeyId,pubKey,personId = getPersonBox(session,wfRequest,personCache,'outbox')
if not feedUrl: if not feedUrl:
return userPosts return userPosts
@ -349,7 +366,7 @@ def threadSendPost(session,postJsonObject: {},federationList: [],inboxUrl: str,b
time.sleep(backoffTime) time.sleep(backoffTime)
backoffTime *= 2 backoffTime *= 2
def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsername: str, toDomain: str, toPort: int, cc: str, https: bool, content: str, followersOnly: bool, saveToFile: bool, federationList: [], sendThreads: [], postLog: [], inReplyTo=None, inReplyToAtomUri=None, subject=None) -> int: def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsername: str, toDomain: str, toPort: int, cc: str, https: bool, content: str, followersOnly: bool, saveToFile: bool, federationList: [], sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {},inReplyTo=None, inReplyToAtomUri=None, subject=None) -> int:
"""Post to another inbox """Post to another inbox
""" """
prefix='https' prefix='https'
@ -364,12 +381,12 @@ def sendPost(session,baseDir: str,username: str, domain: str, port: int, toUsern
handle=prefix+'://'+toDomain+'/@'+toUsername handle=prefix+'://'+toDomain+'/@'+toUsername
# lookup the inbox for the To handle # lookup the inbox for the To handle
wfRequest = webfingerHandle(session,handle,https) wfRequest = webfingerHandle(session,handle,https,cachedWebfingers)
if not wfRequest: if not wfRequest:
return 1 return 1
# get the actor inbox for the To handle # get the actor inbox for the To handle
inboxUrl,pubKeyId,pubKey,toPersonId = getPersonBox(session,wfRequest,'inbox') inboxUrl,pubKeyId,pubKey,toPersonId = getPersonBox(session,wfRequest,personCache,'inbox')
if not inboxUrl: if not inboxUrl:
return 2 return 2
if not pubKey: if not pubKey:

9
tests.py
View File

@ -78,8 +78,9 @@ def testCache():
print('testCache') print('testCache')
personUrl="cat@cardboard.box" personUrl="cat@cardboard.box"
personJson={ "id": 123456, "test": "This is a test" } personJson={ "id": 123456, "test": "This is a test" }
storePersonInCache(personUrl,personJson) personCache={}
result=getPersonFromCache(personUrl) storePersonInCache(personUrl,personJson,personCache)
result=getPersonFromCache(personUrl,personCache)
assert result['id']==123456 assert result['id']==123456
assert result['test']=='This is a test' assert result['test']=='This is a test'
@ -189,7 +190,9 @@ def testPostMessageBetweenServers():
followersOnly=False followersOnly=False
saveToFile=True saveToFile=True
ccUrl=None ccUrl=None
sendResult = sendPost(sessionAlice,aliceDir,'alice', '127.0.0.1', alicePort, 'bob', '127.0.0.1', bobPort, ccUrl, https, 'Why is a mouse when it spins?', followersOnly, saveToFile, federationList, aliceSendThreads, alicePostLog, inReplyTo, inReplyToAtomUri, subject) alicePersonCache={}
aliceCachedWebfingers={}
sendResult = sendPost(sessionAlice,aliceDir,'alice', '127.0.0.1', alicePort, 'bob', '127.0.0.1', bobPort, ccUrl, https, 'Why is a mouse when it spins?', followersOnly, saveToFile, federationList, aliceSendThreads, alicePostLog, aliceCachedWebfingers,alicePersonCache,inReplyTo, inReplyToAtomUri, subject)
print('sendResult: '+str(sendResult)) print('sendResult: '+str(sendResult))
time.sleep(15) time.sleep(15)

6
webfinger.py
View File

@ -34,11 +34,11 @@ def parseHandle(handle: str) -> (str,str):
return username, domain return username, domain
def webfingerHandle(session,handle: str,https: bool) -> {}: def webfingerHandle(session,handle: str,https: bool,cachedWebfingers: {}) -> {}:
username, domain = parseHandle(handle) username, domain = parseHandle(handle)
if not username: if not username:
return None return None
wf=getWebfingerFromCache(username+'@'+domain) wf=getWebfingerFromCache(username+'@'+domain,cachedWebfingers)
if wf: if wf:
return wf return wf
prefix='https' prefix='https'
@ -54,7 +54,7 @@ def webfingerHandle(session,handle: str,https: bool) -> {}:
#except: #except:
# print("Unable to webfinger " + url) # print("Unable to webfinger " + url)
# return None # return None
storeWebfingerInCache(username+'@'+domain, result) storeWebfingerInCache(username+'@'+domain, result,cachedWebfingers)
return result return result
def generateMagicKey(publicKeyPem) -> str: def generateMagicKey(publicKeyPem) -> str: