epicyon/daemon.py

454 lines
18 KiB
Python
Raw Normal View History

2019-06-28 18:55:29 +00:00
__filename__ = "daemon.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "0.0.1"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
2019-07-01 21:01:43 +00:00
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
2019-06-28 18:55:29 +00:00
#import socketserver
import json
2019-07-01 14:30:48 +00:00
import time
2019-06-28 18:55:29 +00:00
from pprint import pprint
from session import createSession
from webfinger import webfingerMeta
from webfinger import webfingerLookup
2019-07-01 21:01:43 +00:00
from webfinger import webfingerHandle
2019-06-28 18:55:29 +00:00
from person import personLookup
2019-07-04 16:24:23 +00:00
from person import personBoxJson
from posts import outboxMessageCreateWrap
2019-07-04 16:24:23 +00:00
from posts import savePostToBox
2019-06-28 21:59:54 +00:00
from inbox import inboxPermittedMessage
2019-07-02 15:07:27 +00:00
from inbox import inboxMessageHasParams
2019-07-04 12:23:53 +00:00
from inbox import runInboxQueue
2019-07-04 14:36:29 +00:00
from inbox import savePostToInboxQueue
2019-06-29 20:21:37 +00:00
from follow import getFollowingFeed
2019-07-03 18:24:44 +00:00
from auth import authorize
2019-07-04 12:23:53 +00:00
from threads import threadWithTrace
2019-06-28 18:55:29 +00:00
import os
2019-06-28 21:06:05 +00:00
import sys
2019-06-28 18:55:29 +00:00
2019-06-28 21:06:05 +00:00
# Avoid giant messages
maxMessageLength=5000
2019-06-29 14:35:26 +00:00
# maximum number of posts to list in outbox feed
maxPostsInFeed=20
2019-06-29 20:21:37 +00:00
# number of follows/followers per page
2019-06-29 20:34:41 +00:00
followsPerPage=12
2019-06-29 20:21:37 +00:00
2019-06-28 18:55:29 +00:00
def readFollowList(filename: str):
"""Returns a list of ActivityPub addresses to follow
"""
followlist=[]
if not os.path.isfile(filename):
return followlist
followUsers = open(filename, "r")
for u in followUsers:
if u not in followlist:
2019-07-03 09:40:27 +00:00
nickname,domain = parseHandle(u)
if nickname:
followlist.append(nickname+'@'+domain)
2019-06-29 20:21:37 +00:00
followUsers.close()
2019-06-28 18:55:29 +00:00
return followlist
class PubServer(BaseHTTPRequestHandler):
def _set_headers(self,fileFormat: str) -> None:
2019-06-28 18:55:29 +00:00
self.send_response(200)
self.send_header('Content-type', fileFormat)
self.end_headers()
def _404(self) -> None:
2019-06-28 18:55:29 +00:00
self.send_response(404)
self.send_header('Content-Type', 'text/html; charset=utf-8')
self.end_headers()
self.wfile.write("<html><head></head><body><h1>404 Not Found</h1></body></html>".encode('utf-8'))
def _webfinger(self) -> bool:
if not self.path.startswith('/.well-known'):
return False
2019-07-04 14:36:29 +00:00
if self.server.debug:
print('DEBUG: WEBFINGER well-known')
2019-06-28 18:55:29 +00:00
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: WEBFINGER host-meta')
2019-06-28 18:55:29 +00:00
if self.path.startswith('/.well-known/host-meta'):
wfResult=webfingerMeta()
if wfResult:
self._set_headers('application/xrd+xml')
self.wfile.write(wfResult.encode('utf-8'))
return
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: WEBFINGER lookup '+self.path+' '+str(self.server.baseDir))
2019-06-30 22:56:37 +00:00
wfResult=webfingerLookup(self.path,self.server.baseDir)
2019-06-28 18:55:29 +00:00
if wfResult:
2019-07-01 21:01:43 +00:00
self._set_headers('application/jrd+json')
2019-06-28 18:55:29 +00:00
self.wfile.write(json.dumps(wfResult).encode('utf-8'))
else:
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: WEBFINGER lookup 404 '+self.path)
2019-06-28 18:55:29 +00:00
self._404()
return True
def _permittedDir(self,path: str) -> bool:
"""These are special paths which should not be accessible
directly via GET or POST
"""
2019-06-28 18:55:29 +00:00
if path.startswith('/wfendpoints') or \
path.startswith('/keys') or \
path.startswith('/accounts'):
return False
return True
2019-07-05 11:41:09 +00:00
def _postToOutbox(self,messageJson: {}) -> bool:
"""post is received by the outbox
Client to server message post
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
"""
2019-07-03 22:59:56 +00:00
if not messageJson.get('type'):
2019-07-04 10:02:56 +00:00
if self.server.debug:
print('DEBUG: POST to outbox has no "type" parameter')
2019-07-03 22:59:56 +00:00
return False
if not messageJson.get('object') and messageJson.get('content'):
if messageJson['type']!='Create':
# https://www.w3.org/TR/activitypub/#object-without-create
if self.server.debug:
2019-07-04 10:02:56 +00:00
print('DEBUG: POST to outbox - adding Create wrapper')
2019-07-03 22:59:56 +00:00
messageJson= \
outboxMessageCreateWrap(self.server.httpPrefix, \
self.postToNickname, \
self.server.domain,messageJson)
2019-07-03 22:44:03 +00:00
if messageJson['type']=='Create':
if not (messageJson.get('id') and \
messageJson.get('type') and \
messageJson.get('actor') and \
messageJson.get('object') and \
messageJson.get('atomUri') and \
messageJson.get('to')):
if self.server.debug:
2019-07-04 10:02:56 +00:00
print('DEBUG: POST to outbox - Create does not have the required parameters')
2019-07-03 22:44:03 +00:00
return False
# https://www.w3.org/TR/activitypub/#create-activity-outbox
messageJson['object']['attributedTo']=messageJson['actor']
permittedOutboxTypes=['Create','Announce','Like','Follow','Undo','Update','Add','Remove','Block','Delete']
if messageJson['type'] not in permittedOutboxTypes:
if self.server.debug:
2019-07-04 10:02:56 +00:00
print('DEBUG: POST to outbox - '+messageJson['type']+' is not a permitted activity type')
return False
2019-07-03 22:59:56 +00:00
if messageJson.get('id'):
postId=messageJson['id']
else:
postId=None
2019-07-04 16:24:23 +00:00
savePostToBox(self.server.baseDir,postId,self.postToNickname,self.server.domain,messageJson,'outbox')
return True
2019-07-05 11:41:09 +00:00
def _updateInboxQueue(self,nickname: str,messageJson: {}) -> bool:
2019-07-05 11:27:18 +00:00
"""Update the inbox queue
"""
cacheFilename = \
savePostToInboxQueue(self.server.baseDir, \
self.server.httpPrefix, \
nickname, \
self.server.domain, \
messageJson,
self.headers['host'],
self.headers['signature'])
if cacheFilename:
if cacheFilename not in self.server.inboxQueue:
self.server.inboxQueue.append(cacheFilename)
self.send_response(201)
self.end_headers()
self.server.POSTbusy=False
return True
return False
2019-07-01 21:01:43 +00:00
def do_GET(self):
2019-07-03 16:14:45 +00:00
if self.server.debug:
2019-07-04 17:56:25 +00:00
print('DEBUG: GET from '+self.server.baseDir+' path: '+self.path+' busy: '+str(self.server.GETbusy))
2019-07-01 14:30:48 +00:00
if self.server.GETbusy:
currTimeGET=int(time.time())
if currTimeGET-self.server.lastGET<10:
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: GET Busy')
2019-06-29 17:28:43 +00:00
self.send_response(429)
2019-06-29 17:25:09 +00:00
self.end_headers()
2019-07-01 14:30:48 +00:00
return
self.server.lastGET=currTimeGET
self.server.GETbusy=True
2019-06-30 20:03:23 +00:00
2019-06-28 21:59:54 +00:00
if not self._permittedDir(self.path):
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: GET Not permitted')
2019-06-28 18:55:29 +00:00
self._404()
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-28 18:55:29 +00:00
return
# get webfinger endpoint for a person
if self._webfinger():
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-28 18:55:29 +00:00
return
2019-07-03 19:32:07 +00:00
# get the inbox for a given person
if self.path.endswith('/inbox'):
if '/users/' in self.path:
if self.headers.get('Authorization'):
2019-07-04 16:24:23 +00:00
if authorize(self.server.baseDir,self.path, \
self.headers['Authorization'], \
self.server.debug):
inboxFeed=personBoxJson(self.server.baseDir, \
self.server.domain, \
self.server.port, \
self.path, \
self.server.httpPrefix, \
maxPostsInFeed, 'inbox')
if inboxFeed:
self._set_headers('application/json')
self.wfile.write(json.dumps(inboxFeed).encode('utf-8'))
self.server.GETbusy=False
return
2019-07-04 08:56:15 +00:00
else:
if self.server.debug:
print('DEBUG: '+nickname+' was not authorized to access '+self.path)
if self.server.debug:
print('DEBUG: GET access to inbox is unauthorized')
2019-07-03 20:32:30 +00:00
self.send_response(405)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-03 19:32:07 +00:00
2019-06-29 14:35:26 +00:00
# get outbox feed for a person
2019-07-04 16:24:23 +00:00
outboxFeed=personBoxJson(self.server.baseDir,self.server.domain, \
self.server.port,self.path, \
self.server.httpPrefix, \
maxPostsInFeed, 'outbox')
2019-06-29 14:41:23 +00:00
if outboxFeed:
2019-06-29 14:35:26 +00:00
self._set_headers('application/json')
2019-06-29 14:41:23 +00:00
self.wfile.write(json.dumps(outboxFeed).encode('utf-8'))
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-29 20:21:37 +00:00
return
2019-07-02 20:54:22 +00:00
following=getFollowingFeed(self.server.baseDir,self.server.domain, \
self.server.port,self.path, \
2019-07-03 19:00:03 +00:00
self.server.httpPrefix,followsPerPage)
2019-06-29 20:21:37 +00:00
if following:
self._set_headers('application/json')
self.wfile.write(json.dumps(following).encode('utf-8'))
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-29 20:21:37 +00:00
return
2019-07-02 20:54:22 +00:00
followers=getFollowingFeed(self.server.baseDir,self.server.domain, \
self.server.port,self.path, \
2019-07-03 19:00:03 +00:00
self.server.httpPrefix,followsPerPage,'followers')
2019-06-29 20:21:37 +00:00
if followers:
self._set_headers('application/json')
self.wfile.write(json.dumps(followers).encode('utf-8'))
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-07-04 14:36:29 +00:00
return
2019-06-28 18:55:29 +00:00
# look up a person
2019-07-04 20:25:19 +00:00
print('************personLookup start '+self.server.domain+' '+self.path+' '+self.server.baseDir)
2019-07-02 20:54:22 +00:00
getPerson = personLookup(self.server.domain,self.path, \
self.server.baseDir)
2019-07-04 20:25:19 +00:00
print('************personLookup end '+str(getPerson))
2019-06-28 18:55:29 +00:00
if getPerson:
self._set_headers('application/json')
self.wfile.write(json.dumps(getPerson).encode('utf-8'))
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-28 18:55:29 +00:00
return
# check that a json file was requested
if not self.path.endswith('.json'):
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: GET Not json: '+self.path+' '+self.server.baseDir)
2019-06-28 18:55:29 +00:00
self._404()
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-28 18:55:29 +00:00
return
# check that the file exists
2019-06-30 22:56:37 +00:00
filename=self.server.baseDir+self.path
2019-06-28 18:55:29 +00:00
if os.path.isfile(filename):
self._set_headers('application/json')
2019-07-03 19:15:42 +00:00
with open(filename, 'r', encoding='utf-8') as File:
2019-06-28 18:55:29 +00:00
content = File.read()
contentJson=json.loads(content)
2019-07-03 19:15:42 +00:00
self.wfile.write(json.dumps(contentJson).encode('utf-8'))
2019-06-28 18:55:29 +00:00
else:
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: GET Unknown file')
2019-06-28 18:55:29 +00:00
self._404()
2019-07-01 14:30:48 +00:00
self.server.GETbusy=False
2019-06-28 18:55:29 +00:00
def do_HEAD(self):
self._set_headers('application/json')
2019-07-01 21:01:43 +00:00
2019-06-28 18:55:29 +00:00
def do_POST(self):
2019-07-04 17:56:25 +00:00
if self.server.debug:
print('DEBUG: POST to from '+self.server.baseDir+' path: '+self.path+' busy: '+str(self.server.POSTbusy))
2019-07-01 14:30:48 +00:00
if self.server.POSTbusy:
currTimePOST=int(time.time())
if currTimePOST-self.server.lastPOST<10:
2019-06-29 17:28:43 +00:00
self.send_response(429)
2019-06-29 17:27:32 +00:00
self.end_headers()
2019-07-01 14:30:48 +00:00
return
self.server.lastPOST=currTimePOST
2019-07-03 16:14:45 +00:00
2019-07-01 14:30:48 +00:00
self.server.POSTbusy=True
2019-07-01 11:48:54 +00:00
if not self.headers.get('Content-type'):
2019-07-03 16:14:45 +00:00
print('Content-type header missing')
2019-07-01 11:48:54 +00:00
self.send_response(400)
self.end_headers()
2019-07-01 14:30:48 +00:00
self.server.POSTbusy=False
2019-07-01 11:48:54 +00:00
return
2019-06-28 18:55:29 +00:00
# refuse to receive non-json content
2019-07-01 11:48:54 +00:00
if self.headers['Content-type'] != 'application/json':
2019-07-03 16:14:45 +00:00
print("POST is not json: "+self.headers['Content-type'])
self.send_response(400)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-03 21:37:46 +00:00
# remove any trailing slashes from the path
2019-07-05 10:21:10 +00:00
self.path=self.path.replace('/outbox/','/outbox').replace('/inbox/','/inbox').replace('/sharedInbox/','/sharedInbox')
2019-07-03 21:37:46 +00:00
# if this is a POST to teh outbox then check authentication
self.outboxAuthenticated=False
self.postToNickname=None
2019-07-04 10:02:56 +00:00
if self.path.endswith('/outbox'):
if '/users/' in self.path:
if self.headers.get('Authorization'):
2019-07-04 09:00:53 +00:00
if authorize(self.server.baseDir,self.path,self.headers['Authorization'],self.server.debug):
self.outboxAuthenticated=True
2019-07-04 10:02:56 +00:00
pathUsersSection=path.split('/users/')[1]
self.postToNickname=pathUsersSection.split('/')[0]
2019-07-03 21:37:46 +00:00
if not self.outboxAuthenticated:
self.send_response(405)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-03 16:14:45 +00:00
# check that the post is to an expected path
2019-07-05 10:21:10 +00:00
if not (self.path.endswith('/outbox') or \
self.path.endswith('/inbox') or \
self.path=='/sharedInbox'):
2019-07-03 16:14:45 +00:00
print('Attempt to POST to invalid path '+self.path)
self.send_response(400)
self.end_headers()
self.server.POSTbusy=False
return
2019-06-28 18:55:29 +00:00
# read the message and convert it into a python dictionary
2019-07-01 11:48:54 +00:00
length = int(self.headers['Content-length'])
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: content-length: '+str(length))
2019-06-28 21:06:05 +00:00
if length>maxMessageLength:
self.send_response(400)
self.end_headers()
2019-07-01 14:30:48 +00:00
self.server.POSTbusy=False
2019-06-28 21:06:05 +00:00
return
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: Reading message')
2019-07-01 11:48:54 +00:00
messageBytes=self.rfile.read(length)
messageJson = json.loads(messageBytes)
2019-06-28 20:35:34 +00:00
2019-07-03 21:37:46 +00:00
# https://www.w3.org/TR/activitypub/#object-without-create
if self.outboxAuthenticated:
if self._postToOutbox(messageJson):
self.send_header('Location',messageJson['object']['atomUri'])
self.send_response(201)
self.end_headers()
self.server.POSTbusy=False
return
else:
self.send_response(403)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-03 21:37:46 +00:00
2019-07-02 15:07:27 +00:00
# check the necessary properties are available
2019-07-03 16:14:45 +00:00
if self.server.debug:
print('DEBUG: Check message has params')
2019-07-05 10:21:10 +00:00
if self.path.endswith('/inbox') or self.path=='/sharedInbox':
2019-07-03 21:37:46 +00:00
if not inboxMessageHasParams(messageJson):
self.send_response(403)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-02 15:07:27 +00:00
2019-07-01 11:48:54 +00:00
if not inboxPermittedMessage(self.server.domain,messageJson,self.server.federationList):
2019-07-03 16:14:45 +00:00
if self.server.debug:
# https://www.youtube.com/watch?v=K3PrSj9XEu4
2019-07-03 16:14:45 +00:00
print('DEBUG: Ah Ah Ah')
2019-06-28 21:59:54 +00:00
self.send_response(403)
self.end_headers()
2019-07-01 14:30:48 +00:00
self.server.POSTbusy=False
2019-07-01 11:48:54 +00:00
return
2019-07-01 14:30:48 +00:00
2019-07-05 10:21:10 +00:00
if self.server.debug:
pprint(messageJson)
2019-07-02 17:11:59 +00:00
2019-07-04 14:36:29 +00:00
if not self.headers.get('signature'):
if 'keyId=' not in self.headers['signature']:
if self.server.debug:
print('DEBUG: POST to inbox has no keyId in header signature parameter')
self.send_response(403)
self.end_headers()
self.server.POSTbusy=False
return
2019-07-04 10:02:56 +00:00
2019-07-03 16:14:45 +00:00
if self.server.debug:
2019-07-04 10:02:56 +00:00
print('DEBUG: POST saving to inbox cache')
if '/users/' in self.path:
pathUsersSection=self.path.split('/users/')[1]
if '/' not in pathUsersSection:
if self.server.debug:
print('DEBUG: This is not a users endpoint')
else:
self.postToNickname=pathUsersSection.split('/')[0]
if self.postToNickname:
2019-07-05 11:41:09 +00:00
if self._updateInboxQueue(self.postToNickname,messageJson):
2019-07-04 12:23:53 +00:00
return
self.send_response(403)
2019-07-01 14:30:48 +00:00
self.end_headers()
self.server.POSTbusy=False
return
2019-07-04 12:23:53 +00:00
else:
2019-07-05 10:24:20 +00:00
if self.path == '/sharedInbox':
print('DEBUG: POST to shared inbox')
2019-07-05 11:41:09 +00:00
if self._updateInboxQueue('sharedinbox',messageJson):
2019-07-05 11:27:18 +00:00
return
2019-07-05 10:24:20 +00:00
self.send_response(200)
self.end_headers()
self.server.POSTbusy=False
2019-07-03 16:14:45 +00:00
2019-07-04 14:36:29 +00:00
def runDaemon(baseDir: str,domain: str,port=80,httpPrefix='https',fedList=[],useTor=False,debug=False) -> None:
2019-06-28 18:55:29 +00:00
if len(domain)==0:
2019-07-03 09:24:55 +00:00
domain='localhost'
2019-06-28 18:55:29 +00:00
if '.' not in domain:
2019-07-03 12:24:54 +00:00
if domain != 'localhost':
print('Invalid domain: ' + domain)
return
2019-06-28 18:55:29 +00:00
2019-07-03 09:24:55 +00:00
serverAddress = ('', port)
2019-07-01 21:01:43 +00:00
httpd = ThreadingHTTPServer(serverAddress, PubServer)
2019-06-30 20:03:23 +00:00
httpd.domain=domain
httpd.port=port
2019-07-03 19:00:03 +00:00
httpd.httpPrefix=httpPrefix
2019-07-03 16:14:45 +00:00
httpd.debug=debug
2019-06-30 20:03:23 +00:00
httpd.federationList=fedList.copy()
2019-07-04 20:25:19 +00:00
httpd.baseDir=baseDir
2019-07-01 14:30:48 +00:00
httpd.personCache={}
httpd.cachedWebfingers={}
httpd.useTor=useTor
2019-07-02 17:11:59 +00:00
httpd.session = None
httpd.sessionLastUpdate=0
2019-07-01 14:30:48 +00:00
httpd.lastGET=0
httpd.lastPOST=0
httpd.GETbusy=False
httpd.POSTbusy=False
2019-07-01 21:01:43 +00:00
httpd.receivedMessage=False
2019-07-04 10:02:56 +00:00
httpd.inboxQueue=[]
2019-06-30 20:03:23 +00:00
print('Running ActivityPub daemon on ' + domain + ' port ' + str(port))
2019-07-04 12:23:53 +00:00
httpd.thrInboxQueue=threadWithTrace(target=runInboxQueue,args=(baseDir,httpPrefix,httpd.personCache,httpd.inboxQueue,domain,port,useTor,httpd.federationList,debug),daemon=True)
httpd.thrInboxQueue.start()
2019-06-28 18:55:29 +00:00
httpd.serve_forever()