epicyon/posts.py

1324 lines
51 KiB
Python
Raw Normal View History

2019-06-28 18:55:29 +00:00
__filename__ = "posts.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "0.0.1"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
import requests
import json
2019-06-29 10:08:59 +00:00
import commentjson
2019-06-28 18:55:29 +00:00
import html
2019-06-29 10:08:59 +00:00
import datetime
2019-06-30 15:03:26 +00:00
import os
import shutil
2019-06-30 13:20:23 +00:00
import threading
2019-06-30 15:03:26 +00:00
import sys
import trace
2019-07-01 11:48:54 +00:00
import time
from collections import OrderedDict
2019-06-30 16:36:58 +00:00
from threads import threadWithTrace
2019-06-30 15:03:26 +00:00
from cache import storePersonInCache
from cache import getPersonFromCache
2019-06-29 10:08:59 +00:00
from pprint import pprint
2019-06-28 18:55:29 +00:00
from random import randint
2019-07-03 10:33:55 +00:00
from session import createSession
2019-06-28 18:55:29 +00:00
from session import getJson
2019-06-30 11:07:39 +00:00
from session import postJson
2019-07-16 14:23:06 +00:00
from session import postImage
2019-06-30 22:56:37 +00:00
from webfinger import webfingerHandle
2019-07-01 09:31:02 +00:00
from httpsig import createSignedHeader
2019-07-02 09:25:29 +00:00
from utils import getStatusNumber
2019-07-04 16:24:23 +00:00
from utils import createPersonDir
2019-07-02 10:39:55 +00:00
from utils import urlPermitted
2019-07-09 14:20:23 +00:00
from utils import getNicknameFromActor
from utils import getDomainFromActor
2019-07-14 16:37:01 +00:00
from utils import deletePost
from capabilities import getOcapFilename
2019-07-09 14:20:23 +00:00
from capabilities import capabilitiesUpdate
2019-07-12 19:08:46 +00:00
from media import attachImage
2019-07-15 14:41:15 +00:00
from content import addMentions
2019-07-16 10:19:04 +00:00
from auth import createBasicAuthHeader
2019-06-28 18:55:29 +00:00
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
from bs4 import BeautifulSoup
2019-07-06 17:00:22 +00:00
def noOfFollowersOnDomain(baseDir: str,handle: str, \
domain: str, followFile='followers.txt') -> int:
2019-07-05 14:39:24 +00:00
"""Returns the number of followers of the given handle from the given domain
"""
filename=baseDir+'/accounts/'+handle+'/'+followFile
if not os.path.isfile(filename):
return 0
ctr=0
with open(filename, "r") as followersFilename:
for followerHandle in followersFilename:
if '@' in followerHandle:
2019-07-06 17:00:22 +00:00
followerDomain= \
followerHandle.split('@')[1].replace('\n','')
2019-07-05 14:39:24 +00:00
if domain==followerDomain:
ctr+=1
return ctr
2019-07-06 17:00:22 +00:00
def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
debug=False):
2019-06-30 15:03:26 +00:00
"""Returns the public or private key of a person
"""
2019-07-03 09:40:27 +00:00
handle=nickname+'@'+domain
2019-06-30 15:03:26 +00:00
keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
if not os.path.isfile(keyFilename):
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: private key file not found: '+keyFilename)
2019-06-30 15:03:26 +00:00
return ''
keyPem=''
with open(keyFilename, "r") as pemFile:
keyPem=pemFile.read()
if len(keyPem)<20:
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: private key was too short: '+keyPem)
2019-06-30 15:03:26 +00:00
return ''
return keyPem
2019-06-28 18:55:29 +00:00
def cleanHtml(rawHtml: str) -> str:
text = BeautifulSoup(rawHtml, 'html.parser').get_text()
return html.unescape(text)
def getUserUrl(wfRequest) -> str:
if wfRequest.get('links'):
for link in wfRequest['links']:
if link.get('type') and link.get('href'):
if link['type'] == 'application/activity+json':
return link['href']
return None
2019-07-01 11:09:09 +00:00
def parseUserFeed(session,feedUrl: str,asHeader: {}) -> None:
2019-06-29 10:08:59 +00:00
feedJson = getJson(session,feedUrl,asHeader,None)
2019-07-04 17:31:41 +00:00
if not feedJson:
return
2019-06-28 18:55:29 +00:00
2019-06-29 10:08:59 +00:00
if 'orderedItems' in feedJson:
2019-06-29 10:59:16 +00:00
for item in feedJson['orderedItems']:
2019-06-28 18:55:29 +00:00
yield item
nextUrl = None
2019-06-29 10:08:59 +00:00
if 'first' in feedJson:
2019-06-29 10:59:16 +00:00
nextUrl = feedJson['first']
2019-06-29 10:08:59 +00:00
elif 'next' in feedJson:
nextUrl = feedJson['next']
2019-06-28 18:55:29 +00:00
if nextUrl:
2019-06-28 19:36:39 +00:00
for item in parseUserFeed(session,nextUrl,asHeader):
2019-06-28 18:55:29 +00:00
yield item
2019-06-30 11:34:19 +00:00
2019-07-06 17:00:22 +00:00
def getPersonBox(session,wfRequest: {},personCache: {}, \
boxName='inbox') -> (str,str,str,str,str):
2019-06-30 10:14:02 +00:00
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
personUrl = getUserUrl(wfRequest)
if not personUrl:
2019-07-05 13:50:27 +00:00
return None,None,None,None,None
2019-07-01 14:30:48 +00:00
personJson = getPersonFromCache(personUrl,personCache)
2019-06-30 11:34:19 +00:00
if not personJson:
2019-06-30 10:21:07 +00:00
personJson = getJson(session,personUrl,asHeader,None)
2019-07-04 17:31:41 +00:00
if not personJson:
2019-07-05 13:50:27 +00:00
return None,None,None,None,None
2019-07-05 13:38:29 +00:00
boxJson=None
2019-06-30 10:14:02 +00:00
if not personJson.get(boxName):
2019-07-05 13:38:29 +00:00
if personJson.get('endpoints'):
if personJson['endpoints'].get(boxName):
boxJson=personJson['endpoints'][boxName]
else:
boxJson=personJson[boxName]
if not boxJson:
2019-07-05 13:50:27 +00:00
return None,None,None,None,None
2019-07-05 13:38:29 +00:00
2019-06-30 10:14:02 +00:00
personId=None
if personJson.get('id'):
personId=personJson['id']
2019-07-01 10:25:03 +00:00
pubKeyId=None
2019-06-30 10:14:02 +00:00
pubKey=None
if personJson.get('publicKey'):
2019-07-01 10:25:03 +00:00
if personJson['publicKey'].get('id'):
pubKeyId=personJson['publicKey']['id']
2019-06-30 10:14:02 +00:00
if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem']
2019-07-05 13:50:27 +00:00
sharedInbox=None
if personJson.get('sharedInbox'):
sharedInbox=personJson['sharedInbox']
else:
if personJson.get('endpoints'):
if personJson['endpoints'].get('sharedInbox'):
sharedInbox=personJson['endpoints']['sharedInbox']
2019-07-05 20:32:21 +00:00
capabilityAcquisition=None
2019-07-06 17:00:22 +00:00
if personJson.get('capabilityAcquisitionEndpoint'):
capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
2019-06-30 10:21:07 +00:00
2019-07-01 14:30:48 +00:00
storePersonInCache(personUrl,personJson,personCache)
2019-06-30 10:21:07 +00:00
2019-07-05 20:32:21 +00:00
return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition
2019-06-30 10:14:02 +00:00
2019-07-02 20:54:22 +00:00
def getPosts(session,outboxUrl: str,maxPosts: int,maxMentions: int, \
2019-07-06 10:33:57 +00:00
maxEmoji: int,maxAttachments: int, \
2019-07-09 14:20:23 +00:00
federationList: [],\
2019-07-03 11:24:38 +00:00
personCache: {},raw: bool,simple: bool) -> {}:
2019-07-02 09:25:29 +00:00
personPosts={}
if not outboxUrl:
return personPosts
2019-06-28 18:55:29 +00:00
2019-07-02 09:25:29 +00:00
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
2019-07-03 11:24:38 +00:00
if raw:
result = []
i = 0
for item in parseUserFeed(session,outboxUrl,asHeader):
result.append(item)
i += 1
if i == maxPosts:
break
pprint(result)
return None
2019-06-28 18:55:29 +00:00
i = 0
2019-07-02 09:25:29 +00:00
for item in parseUserFeed(session,outboxUrl,asHeader):
2019-06-28 18:55:29 +00:00
if not item.get('type'):
continue
if item['type'] != 'Create':
continue
if not item.get('object'):
continue
published = item['object']['published']
2019-07-02 09:25:29 +00:00
if not personPosts.get(published):
2019-06-28 18:55:29 +00:00
content = item['object']['content']
mentions=[]
emoji={}
if item['object'].get('tag'):
for tagItem in item['object']['tag']:
tagType=tagItem['type'].lower()
if tagType=='emoji':
if tagItem.get('name') and tagItem.get('icon'):
if tagItem['icon'].get('url'):
# No emoji from non-permitted domains
2019-07-06 17:00:22 +00:00
if urlPermitted(tagItem['icon']['url'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-06 17:00:22 +00:00
"objects:read"):
2019-06-28 18:55:29 +00:00
emojiName=tagItem['name']
emojiIcon=tagItem['icon']['url']
emoji[emojiName]=emojiIcon
if tagType=='mention':
if tagItem.get('name'):
if tagItem['name'] not in mentions:
mentions.append(tagItem['name'])
if len(mentions)>maxMentions:
continue
if len(emoji)>maxEmoji:
continue
summary = ''
if item['object'].get('summary'):
if item['object']['summary']:
summary = item['object']['summary']
inReplyTo = ''
if item['object'].get('inReplyTo'):
if item['object']['inReplyTo']:
# No replies to non-permitted domains
2019-07-06 17:00:22 +00:00
if not urlPermitted(item['object']['inReplyTo'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-06 17:00:22 +00:00
"objects:read"):
2019-06-28 18:55:29 +00:00
continue
inReplyTo = item['object']['inReplyTo']
conversation = ''
if item['object'].get('conversation'):
if item['object']['conversation']:
# no conversations originated in non-permitted domains
2019-07-06 17:00:22 +00:00
if urlPermitted(item['object']['conversation'], \
2019-07-09 14:20:23 +00:00
federationList,"objects:read"):
2019-06-28 18:55:29 +00:00
conversation = item['object']['conversation']
attachment = []
if item['object'].get('attachment'):
if item['object']['attachment']:
for attach in item['object']['attachment']:
if attach.get('name') and attach.get('url'):
# no attachments from non-permitted domains
2019-07-06 17:00:22 +00:00
if urlPermitted(attach['url'], \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-06 17:00:22 +00:00
"objects:read"):
2019-06-28 18:55:29 +00:00
attachment.append([attach['name'],attach['url']])
sensitive = False
if item['object'].get('sensitive'):
sensitive = item['object']['sensitive']
2019-07-03 11:24:38 +00:00
if simple:
print(cleanHtml(content)+'\n')
else:
personPosts[published] = {
"sensitive": sensitive,
"inreplyto": inReplyTo,
"summary": summary,
"html": content,
"plaintext": cleanHtml(content),
"attachment": attachment,
"mentions": mentions,
"emoji": emoji,
"conversation": conversation
}
2019-06-28 18:55:29 +00:00
i += 1
if i == maxPosts:
break
2019-07-02 09:25:29 +00:00
return personPosts
2019-06-29 10:08:59 +00:00
2019-07-04 16:24:23 +00:00
def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
"""Deletes all posts for a person from inbox or outbox
2019-06-29 11:47:33 +00:00
"""
2019-07-04 16:24:23 +00:00
if boxname!='inbox' and boxname!='outbox':
return
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
for deleteFilename in os.listdir(boxDir):
filePath = os.path.join(boxDir, deleteFilename)
2019-06-29 11:47:33 +00:00
try:
if os.path.isfile(filePath):
os.unlink(filePath)
elif os.path.isdir(filePath): shutil.rmtree(filePath)
except Exception as e:
print(e)
2019-07-06 17:00:22 +00:00
def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
2019-07-14 16:57:06 +00:00
nickname: str, domain: str,postJsonObject: {}, \
2019-07-06 17:00:22 +00:00
boxname: str) -> None:
2019-07-04 16:24:23 +00:00
"""Saves the give json to the give box
"""
2019-07-04 16:24:23 +00:00
if boxname!='inbox' and boxname!='outbox':
return
if ':' in domain:
domain=domain.split(':')[0]
2019-07-04 16:24:23 +00:00
2019-07-03 22:59:56 +00:00
if not postId:
statusNumber,published = getStatusNumber()
2019-07-06 17:00:22 +00:00
postId=httpPrefix+'://'+domain+'/users/'+nickname+ \
'/statuses/'+statusNumber
2019-07-14 16:57:06 +00:00
postJsonObject['id']=postId+'/activity'
if postJsonObject.get('object'):
2019-07-16 19:07:45 +00:00
if isinstance(postJsonObject['object'], dict):
postJsonObject['object']['id']=postId
postJsonObject['object']['atomUri']=postId
2019-07-03 22:59:56 +00:00
2019-07-04 16:24:23 +00:00
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
filename=boxDir+'/'+postId.replace('/','#')+'.json'
with open(filename, 'w') as fp:
2019-07-14 16:57:06 +00:00
commentjson.dump(postJsonObject, fp, indent=4, sort_keys=False)
2019-07-03 09:40:27 +00:00
def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
2019-07-03 19:00:03 +00:00
toUrl: str, ccUrl: str, httpPrefix: str, content: str, \
2019-07-12 19:08:46 +00:00
followersOnly: bool, saveToFile: bool, clientToServer: bool,
attachImageFilename: str,imageDescription: str,useBlurhash: bool, \
2019-07-02 20:54:22 +00:00
inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
2019-07-01 12:14:49 +00:00
"""Creates a message
2019-06-29 22:29:18 +00:00
"""
2019-07-16 10:19:04 +00:00
if not clientToServer:
# convert content to html
content=addMentions(baseDir,httpPrefix, \
nickname,domain,content)
2019-07-15 14:41:15 +00:00
2019-07-01 12:47:08 +00:00
if port!=80 and port!=443:
domain=domain+':'+str(port)
2019-06-29 22:29:18 +00:00
statusNumber,published = getStatusNumber()
conversationDate=published.split('T')[0]
2019-06-29 10:08:59 +00:00
conversationId=statusNumber
2019-06-28 18:55:29 +00:00
postTo='https://www.w3.org/ns/activitystreams#Public'
2019-07-03 19:00:03 +00:00
postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
2019-06-28 18:55:29 +00:00
if followersOnly:
postTo=postCC
postCC=''
2019-07-03 19:00:03 +00:00
newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
2019-07-07 11:53:32 +00:00
2019-06-29 10:23:40 +00:00
sensitive=False
2019-06-30 21:20:02 +00:00
summary=None
2019-06-29 10:23:40 +00:00
if subject:
summary=subject
sensitive=True
2019-07-15 14:41:15 +00:00
2019-07-03 15:10:18 +00:00
if not clientToServer:
2019-07-06 10:33:57 +00:00
actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
2019-07-07 11:53:32 +00:00
# if capabilities have been granted for this actor
# then get the corresponding id
capabilityId=None
2019-07-08 13:30:04 +00:00
capabilityIdList=[]
ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
2019-07-07 11:53:32 +00:00
if os.path.isfile(ocapFilename):
with open(ocapFilename, 'r') as fp:
oc=commentjson.load(fp)
if oc.get('id'):
2019-07-08 13:30:04 +00:00
capabilityIdList=[oc['id']]
2019-07-07 11:53:32 +00:00
2019-07-03 15:10:18 +00:00
newPost = {
'id': newPostId+'/activity',
2019-07-08 13:30:04 +00:00
'capability': capabilityIdList,
2019-07-03 15:10:18 +00:00
'type': 'Create',
2019-07-06 10:33:57 +00:00
'actor': actorUrl,
2019-07-03 15:10:18 +00:00
'published': published,
'to': [toUrl],
'cc': [],
'object': {
'id': newPostId,
'type': 'Note',
'summary': summary,
'inReplyTo': inReplyTo,
'published': published,
2019-07-03 19:00:03 +00:00
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
2019-07-03 15:10:18 +00:00
'to': [toUrl],
'cc': [],
'sensitive': sensitive,
2019-07-03 22:59:56 +00:00
'atomUri': newPostId,
2019-07-03 15:10:18 +00:00
'inReplyToAtomUri': inReplyToAtomUri,
'conversation': 'tag:'+domain+','+conversationDate+':objectId='+conversationId+':objectType=Conversation',
'content': content,
'contentMap': {
'en': content
},
'attachment': [],
'tag': [],
2019-07-11 13:46:12 +00:00
'replies': {
'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'items': []
}
}
2019-07-03 15:10:18 +00:00
}
}
2019-07-12 19:08:46 +00:00
if attachImageFilename:
newPost['object']= \
attachImage(baseDir,httpPrefix,domain,port, \
2019-07-12 19:26:54 +00:00
newPost['object'],attachImageFilename, \
2019-07-12 19:08:46 +00:00
imageDescription,useBlurhash)
2019-07-03 15:10:18 +00:00
else:
newPost = {
'id': newPostId,
'type': 'Note',
'summary': summary,
'inReplyTo': inReplyTo,
'published': published,
2019-07-03 19:00:03 +00:00
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
2019-07-03 15:10:18 +00:00
'to': [toUrl],
'cc': [],
'sensitive': sensitive,
2019-07-03 22:59:56 +00:00
'atomUri': newPostId,
2019-07-03 15:10:18 +00:00
'inReplyToAtomUri': inReplyToAtomUri,
'conversation': 'tag:'+domain+','+conversationDate+':objectId='+conversationId+':objectType=Conversation',
'content': content,
'contentMap': {
'en': content
},
'attachment': [],
'tag': [],
2019-07-12 19:08:46 +00:00
'replies': {
'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'items': []
}
}
2019-06-28 18:55:29 +00:00
}
2019-07-12 19:08:46 +00:00
if attachImageFilename:
newPost= \
attachImage(baseDir,httpPrefix,domain,port, \
2019-07-12 19:26:54 +00:00
newPost,attachImageFilename, \
2019-07-12 19:08:46 +00:00
imageDescription,useBlurhash)
2019-07-01 12:14:49 +00:00
if ccUrl:
if len(ccUrl)>0:
newPost['cc']=ccUrl
newPost['object']['cc']=ccUrl
2019-06-29 10:08:59 +00:00
if saveToFile:
2019-07-06 17:00:22 +00:00
savePostToBox(baseDir,httpPrefix,newPostId, \
nickname,domain,newPost,'outbox')
2019-06-28 18:55:29 +00:00
return newPost
2019-06-29 10:08:59 +00:00
2019-07-16 10:19:04 +00:00
def outboxMessageCreateWrap(httpPrefix: str, \
nickname: str,domain: str,port: int, \
2019-07-06 17:00:22 +00:00
messageJson: {}) -> {}:
2019-07-03 21:37:46 +00:00
"""Wraps a received message in a Create
https://www.w3.org/TR/activitypub/#object-without-create
"""
2019-07-16 10:19:04 +00:00
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
2019-07-03 21:37:46 +00:00
statusNumber,published = getStatusNumber()
if messageJson.get('published'):
published = messageJson['published']
newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
cc=[]
if messageJson.get('cc'):
cc=messageJson['cc']
2019-07-05 20:46:47 +00:00
# TODO
2019-07-16 10:19:04 +00:00
capabilityUrl=[]
2019-07-03 21:37:46 +00:00
newPost = {
'id': newPostId+'/activity',
2019-07-05 20:46:47 +00:00
'capability': capabilityUrl,
2019-07-03 21:37:46 +00:00
'type': 'Create',
'actor': httpPrefix+'://'+domain+'/users/'+nickname,
'published': published,
'to': messageJson['to'],
'cc': cc,
'object': messageJson
}
newPost['object']['id']=newPost['id']
2019-07-06 17:00:22 +00:00
newPost['object']['url']= \
httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
newPost['object']['atomUri']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
2019-07-03 21:37:46 +00:00
return newPost
2019-07-08 13:30:04 +00:00
def postIsAddressedToFollowers(baseDir: str,
nickname: str, domain: str, port: int,httpPrefix: str,
2019-07-14 16:57:06 +00:00
postJsonObject: {}) -> bool:
2019-07-08 13:30:04 +00:00
"""Returns true if the given post is addressed to followers of the nickname
"""
if port!=80 and port!=443:
domain=domain+':'+str(port)
2019-07-14 16:57:06 +00:00
if not postJsonObject.get('object'):
2019-07-08 13:30:04 +00:00
return False
2019-07-16 19:07:45 +00:00
toList=[]
ccList=[]
if isinstance(postJsonObject['object'], dict):
if not postJsonObject['object'].get('to'):
return False
toList=postJsonObject['object']['to']
if postJsonObject['object'].get('cc'):
ccList=postJsonObject['object']['cc']
else:
if not postJsonObject.get('to'):
return False
toList=postJsonObject['to']
if postJsonObject.get('cc'):
ccList=postJsonObject['cc']
2019-07-08 13:30:04 +00:00
followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
# does the followers url exist in 'to' or 'cc' lists?
addressedToFollowers=False
2019-07-16 19:07:45 +00:00
if followersUrl in toList:
2019-07-08 13:30:04 +00:00
addressedToFollowers=True
if not addressedToFollowers:
2019-07-16 19:07:45 +00:00
if followersUrl in ccList:
2019-07-08 13:30:04 +00:00
addressedToFollowers=True
return addressedToFollowers
2019-07-15 17:22:51 +00:00
def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
"""Returns true if the given post is addressed to public
"""
if not postJsonObject.get('object'):
return False
if not postJsonObject['object'].get('to'):
return False
publicUrl='https://www.w3.org/ns/activitystreams#Public'
# does the public url exist in 'to' or 'cc' lists?
addressedToPublic=False
if publicUrl in postJsonObject['object']['to']:
addressedToPublic=True
if not addressedToPublic:
if not postJsonObject['object'].get('cc'):
return False
if publicUrl in postJsonObject['object']['cc']:
addressedToPublic=True
return addressedToPublic
2019-07-03 15:10:18 +00:00
def createPublicPost(baseDir: str,
2019-07-03 19:00:03 +00:00
nickname: str, domain: str, port: int,httpPrefix: str, \
2019-07-03 15:10:18 +00:00
content: str, followersOnly: bool, saveToFile: bool,
2019-07-09 14:20:23 +00:00
clientToServer: bool,\
2019-07-12 19:08:46 +00:00
attachImageFilename: str,imageDescription: str,useBlurhash: bool, \
2019-07-02 20:54:22 +00:00
inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
2019-06-30 10:14:02 +00:00
"""Public post to the outbox
"""
2019-07-03 09:40:27 +00:00
return createPostBase(baseDir,nickname, domain, port, \
2019-07-02 20:54:22 +00:00
'https://www.w3.org/ns/activitystreams#Public', \
2019-07-03 19:00:03 +00:00
httpPrefix+'://'+domain+'/users/'+nickname+'/followers', \
2019-07-06 17:00:22 +00:00
httpPrefix, content, followersOnly, saveToFile, \
2019-07-09 14:20:23 +00:00
clientToServer, \
2019-07-12 19:08:46 +00:00
attachImageFilename,imageDescription,useBlurhash, \
2019-07-02 20:54:22 +00:00
inReplyTo, inReplyToAtomUri, subject)
2019-07-09 14:20:23 +00:00
def threadSendPost(session,postJsonObject: {},federationList: [],\
2019-07-06 13:49:25 +00:00
inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
debug :bool) -> None:
2019-06-30 13:38:01 +00:00
"""Sends a post with exponential backoff
"""
2019-06-30 13:20:23 +00:00
tries=0
2019-06-30 13:38:01 +00:00
backoffTime=60
for attempt in range(20):
2019-07-02 20:54:22 +00:00
postResult = postJson(session,postJsonObject,federationList, \
2019-07-09 14:20:23 +00:00
inboxUrl,signatureHeaderJson, \
2019-07-06 11:03:51 +00:00
"inbox:write")
2019-06-30 13:38:01 +00:00
if postResult:
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: json post to '+inboxUrl+' succeeded')
2019-07-15 12:27:26 +00:00
if postJsonObject.get('published'):
postLog.append(postJsonObject['published']+' '+postResult+'\n')
2019-06-30 13:38:01 +00:00
# keep the length of the log finite
# Don't accumulate massive files on systems with limited resources
while len(postLog)>64:
postlog.pop(0)
# save the log file
filename=baseDir+'/post.log'
with open(filename, "w") as logFile:
for line in postLog:
print(line, file=logFile)
# our work here is done
2019-06-30 13:20:23 +00:00
break
2019-07-06 13:49:25 +00:00
if debug:
2019-07-06 17:00:22 +00:00
print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
str(backoffTime)+' seconds.')
2019-06-30 13:20:23 +00:00
time.sleep(backoffTime)
backoffTime *= 2
2019-07-03 09:40:27 +00:00
def sendPost(session,baseDir: str,nickname: str, domain: str, port: int, \
toNickname: str, toDomain: str, toPort: int, cc: str, \
2019-07-03 19:00:03 +00:00
httpPrefix: str, content: str, followersOnly: bool, \
2019-07-06 10:33:57 +00:00
saveToFile: bool, clientToServer: bool, \
2019-07-12 19:08:46 +00:00
attachImageFilename: str,imageDescription: str,useBlurhash: bool, \
2019-07-09 14:20:23 +00:00
federationList: [],\
2019-07-03 15:10:18 +00:00
sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
2019-07-06 13:49:25 +00:00
debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
2019-06-30 10:14:02 +00:00
"""Post to another inbox
"""
2019-07-01 09:31:02 +00:00
withDigest=True
2019-06-30 22:56:37 +00:00
if toPort!=80 and toPort!=443:
2019-07-06 13:49:25 +00:00
if ':' not in toDomain:
toDomain=toDomain+':'+str(toPort)
2019-06-30 22:56:37 +00:00
2019-07-03 19:00:03 +00:00
handle=httpPrefix+'://'+toDomain+'/@'+toNickname
2019-06-30 22:56:37 +00:00
# lookup the inbox for the To handle
2019-07-03 19:00:03 +00:00
wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers)
2019-06-30 10:14:02 +00:00
if not wfRequest:
return 1
2019-07-05 22:13:20 +00:00
if not clientToServer:
postToBox='inbox'
else:
postToBox='outbox'
2019-06-30 22:56:37 +00:00
# get the actor inbox for the To handle
2019-07-05 20:32:21 +00:00
inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition = \
2019-07-05 22:13:20 +00:00
getPersonBox(session,wfRequest,personCache,postToBox)
2019-07-05 14:39:24 +00:00
# If there are more than one followers on the target domain
2019-07-16 10:19:04 +00:00
# then send to the shared inbox indead of the individual inbox
2019-07-05 22:13:20 +00:00
if nickname=='capabilities':
inboxUrl=capabilityAcquisition
if not capabilityAcquisition:
return 2
else:
if noOfFollowersOnDomain(baseDir,handle,toDomain)>1 and sharedInbox:
inboxUrl=sharedInbox
2019-07-05 14:39:24 +00:00
2019-06-30 10:14:02 +00:00
if not inboxUrl:
return 3
2019-07-05 22:13:20 +00:00
if not pubKey:
2019-06-30 10:14:02 +00:00
return 4
2019-07-05 22:13:20 +00:00
if not toPersonId:
return 5
# sharedInbox and capabilities are optional
2019-06-30 10:14:02 +00:00
2019-07-03 15:10:18 +00:00
postJsonObject = \
createPostBase(baseDir,nickname,domain,port, \
2019-07-03 19:00:03 +00:00
toPersonId,cc,httpPrefix,content, \
2019-07-03 15:10:18 +00:00
followersOnly,saveToFile,clientToServer, \
2019-07-12 19:08:46 +00:00
attachImageFilename,imageDescription,useBlurhash, \
2019-07-06 10:33:57 +00:00
inReplyTo,inReplyToAtomUri,subject)
2019-06-30 10:14:02 +00:00
2019-06-30 22:56:37 +00:00
# get the senders private key
2019-07-03 09:40:27 +00:00
privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
2019-06-30 10:14:02 +00:00
if len(privateKeyPem)==0:
2019-07-05 22:13:20 +00:00
return 6
2019-06-30 10:14:02 +00:00
2019-07-05 22:13:20 +00:00
if toDomain not in inboxUrl:
return 7
postPath='/'+inboxUrl.split('/')[-1]
2019-07-03 15:10:18 +00:00
2019-06-30 11:07:39 +00:00
# construct the http header
2019-07-02 20:54:22 +00:00
signatureHeaderJson = \
2019-07-03 09:40:27 +00:00
createSignedHeader(privateKeyPem, nickname, domain, port, \
2019-07-03 19:00:03 +00:00
postPath, httpPrefix, withDigest, postJsonObject)
2019-07-05 18:57:19 +00:00
# Keep the number of threads being used small
while len(sendThreads)>10:
sendThreads[0].kill()
sendThreads.pop(0)
thr = threadWithTrace(target=threadSendPost,args=(session, \
postJsonObject.copy(), \
federationList, \
inboxUrl,baseDir, \
signatureHeaderJson.copy(), \
2019-07-06 13:49:25 +00:00
postLog,
debug),daemon=True)
2019-07-05 18:57:19 +00:00
sendThreads.append(thr)
thr.start()
return 0
2019-07-16 10:19:04 +00:00
def sendPostViaServer(session,fromNickname: str,password: str, \
fromDomain: str, fromPort: int, \
toNickname: str, toDomain: str, toPort: int, cc: str, \
httpPrefix: str, content: str, followersOnly: bool, \
attachImageFilename: str,imageDescription: str,useBlurhash: bool, \
cachedWebfingers: {},personCache: {}, \
debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
"""Send a post via a proxy (c2s)
"""
2019-07-16 11:33:40 +00:00
if not session:
print('WARN: No session for sendPostViaServer')
return 6
2019-07-16 10:19:04 +00:00
withDigest=True
if toPort!=80 and toPort!=443:
if ':' not in fromDomain:
fromDomain=fromDomain+':'+str(fromPort)
handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
# lookup the inbox for the To handle
wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers)
if not wfRequest:
if debug:
print('DEBUG: webfinger failed for '+handle)
return 1
postToBox='outbox'
# get the actor inbox for the To handle
inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition = \
getPersonBox(session,wfRequest,personCache,postToBox)
if not inboxUrl:
if debug:
print('DEBUG: No '+postToBox+' was found for '+handle)
return 3
if not fromPersonId:
if debug:
print('DEBUG: No actor was found for '+handle)
return 4
# Get the json for the c2s post, not saving anything to file
# Note that baseDir is set to None
saveToFile=False
clientToServer=True
2019-07-17 14:43:51 +00:00
if toDomain.lower().endswith('public'):
toPersonId='https://www.w3.org/ns/activitystreams#Public'
fromDomainFull=fromDomain
if fromPort:
if fromPort!=80 and fromPort!=443:
fromDomainFull=fromDomain+':'+str(fromPort)
cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
else:
if toDomain.lower().endswith('followers') or \
toDomain.lower().endswith('followersonly'):
toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
else:
toDomainFull=toDomain
if toPort!=80 and toDomain!=443:
toDomainFull=toDomain+':'+str(toPort)
toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
2019-07-16 10:19:04 +00:00
postJsonObject = \
createPostBase(None, \
fromNickname,fromDomain,fromPort, \
toPersonId,cc,httpPrefix,content, \
followersOnly,saveToFile,clientToServer, \
attachImageFilename,imageDescription,useBlurhash, \
inReplyTo,inReplyToAtomUri,subject)
2019-07-16 14:23:06 +00:00
2019-07-16 10:19:04 +00:00
authHeader=createBasicAuthHeader(fromNickname,password)
2019-07-16 14:23:06 +00:00
if attachImageFilename:
headers = {'host': fromDomain, \
'Authorization': authHeader}
postResult = \
postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
#if not postResult:
# if debug:
# print('DEBUG: Failed to upload image')
# return 9
2019-07-16 10:19:04 +00:00
headers = {'host': fromDomain, \
'Content-type': 'application/json', \
'Authorization': authHeader}
postResult = \
postJson(session,postJsonObject,[],inboxUrl,headers,"inbox:write")
2019-07-16 14:23:06 +00:00
#if not postResult:
# if debug:
# print('DEBUG: POST failed for c2s to '+inboxUrl)
# return 5
2019-07-16 10:19:04 +00:00
if debug:
print('DEBUG: c2s POST success')
return 0
def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
"""Returns a dictionary with followers grouped by domain
"""
handle=nickname+'@'+domain
followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
if not os.path.isfile(followersFilename):
return None
grouped={}
with open(followersFilename, "r") as f:
for followerHandle in f:
if '@' in followerHandle:
fHandle=followerHandle.strip().replace('\n','')
followerDomain=fHandle.split('@')[1]
if not grouped.get(followerDomain):
grouped[followerDomain]=[fHandle]
else:
grouped[followerDomain].append(fHandle)
return grouped
2019-07-06 17:00:22 +00:00
def sendSignedJson(postJsonObject: {},session,baseDir: str, \
nickname: str, domain: str, port: int, \
2019-07-05 18:57:19 +00:00
toNickname: str, toDomain: str, toPort: int, cc: str, \
2019-07-06 10:33:57 +00:00
httpPrefix: str, saveToFile: bool, clientToServer: bool, \
2019-07-09 14:20:23 +00:00
federationList: [], \
2019-07-06 17:00:22 +00:00
sendThreads: [], postLog: [], cachedWebfingers: {}, \
personCache: {}, debug: bool) -> int:
2019-07-05 18:57:19 +00:00
"""Sends a signed json object to an inbox/outbox
"""
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: sendSignedJson start')
2019-07-16 10:19:04 +00:00
if not session:
print('WARN: No session specified for sendSignedJson')
return 8
2019-07-05 18:57:19 +00:00
withDigest=True
2019-07-08 13:30:04 +00:00
sharedInbox=False
if toNickname=='inbox':
sharedInbox=True
2019-07-05 18:57:19 +00:00
if toPort!=80 and toPort!=443:
2019-07-06 13:49:25 +00:00
if ':' not in toDomain:
toDomain=toDomain+':'+str(toPort)
2019-07-05 18:57:19 +00:00
handle=httpPrefix+'://'+toDomain+'/@'+toNickname
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: handle - '+handle+' toPort '+str(toPort))
2019-07-05 18:57:19 +00:00
# lookup the inbox for the To handle
2019-07-08 13:30:04 +00:00
wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers)
2019-07-05 18:57:19 +00:00
if not wfRequest:
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: webfinger for '+handle+' failed')
2019-07-05 18:57:19 +00:00
return 1
2019-07-05 22:13:20 +00:00
if not clientToServer:
postToBox='inbox'
else:
postToBox='outbox'
# get the actor inbox/outbox/capabilities for the To handle
2019-07-08 13:30:04 +00:00
inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition = \
2019-07-05 22:13:20 +00:00
getPersonBox(session,wfRequest,personCache,postToBox)
2019-07-05 18:57:19 +00:00
2019-07-05 22:13:20 +00:00
if nickname=='capabilities':
inboxUrl=capabilityAcquisition
if not capabilityAcquisition:
return 2
else:
2019-07-08 13:30:04 +00:00
if sharedInbox and sharedInboxUrl:
inboxUrl=sharedInboxUrl
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: Sending to endpoint '+inboxUrl)
2019-07-05 18:57:19 +00:00
if not inboxUrl:
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: missing inboxUrl')
2019-07-05 18:57:19 +00:00
return 3
2019-07-05 22:13:20 +00:00
if not pubKey:
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: missing pubkey')
2019-07-05 18:57:19 +00:00
return 4
2019-07-05 22:13:20 +00:00
if not toPersonId:
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: missing personId')
2019-07-05 22:13:20 +00:00
return 5
# sharedInbox and capabilities are optional
2019-07-05 18:57:19 +00:00
# get the senders private key
2019-07-06 13:49:25 +00:00
privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
2019-07-05 18:57:19 +00:00
if len(privateKeyPem)==0:
2019-07-06 13:49:25 +00:00
if debug:
print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
2019-07-05 22:13:20 +00:00
return 6
2019-07-05 18:57:19 +00:00
2019-07-05 22:13:20 +00:00
if toDomain not in inboxUrl:
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: '+toDomain+' not in '+inboxUrl)
2019-07-05 22:13:20 +00:00
return 7
postPath='/'+inboxUrl.split('/')[-1]
2019-07-05 18:57:19 +00:00
# construct the http header
signatureHeaderJson = \
createSignedHeader(privateKeyPem, nickname, domain, port, \
postPath, httpPrefix, withDigest, postJsonObject)
2019-07-01 09:59:57 +00:00
2019-06-30 13:20:23 +00:00
# Keep the number of threads being used small
2019-06-30 13:42:45 +00:00
while len(sendThreads)>10:
2019-06-30 15:03:26 +00:00
sendThreads[0].kill()
2019-06-30 13:38:01 +00:00
sendThreads.pop(0)
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: starting thread to send post')
2019-07-06 17:00:22 +00:00
thr = threadWithTrace(target=threadSendPost, \
args=(session, \
postJsonObject.copy(), \
federationList, \
inboxUrl,baseDir, \
signatureHeaderJson.copy(), \
postLog,
debug),daemon=True)
2019-06-30 13:20:23 +00:00
sendThreads.append(thr)
thr.start()
2019-06-30 10:14:02 +00:00
return 0
2019-07-15 18:20:52 +00:00
def sendToNamedAddresses(session,baseDir: str, \
nickname: str, domain: str, port: int, \
httpPrefix: str,federationList: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
postJsonObject: {},debug: bool) -> None:
"""sends a post to the specific named addresses in to/cc
"""
2019-07-16 10:19:04 +00:00
if not session:
print('WARN: No session for sendToNamedAddresses')
return
2019-07-15 18:20:52 +00:00
if not postJsonObject.get('object'):
2019-07-16 10:19:04 +00:00
return
2019-07-16 19:07:45 +00:00
toList=[]
if isinstance(postJsonObject['object'], dict):
if not postJsonObject['object'].get('to'):
return
toList=postJsonObject['object']['to']
recipientsObject=postJsonObject['object']
else:
if not postJsonObject.get('to'):
return
toList=postJsonObject['to']
recipientsObject=postJsonObject
2019-07-15 18:20:52 +00:00
recipients=[]
recipientType=['to','cc']
for rType in recipientType:
2019-07-16 19:07:45 +00:00
for address in recipientsObject[rType]:
2019-07-15 18:20:52 +00:00
if address.endswith('#Public'):
continue
if address.endswith('/followers'):
continue
recipients.append(address)
if not recipients:
return
2019-07-15 18:29:30 +00:00
if debug:
2019-07-16 10:19:04 +00:00
print('Sending individually addressed posts: '+str(recipients))
2019-07-15 18:29:30 +00:00
# this is after the message has arrived at the server
2019-07-15 18:20:52 +00:00
clientToServer=False
for address in recipients:
toNickname=getNicknameFromActor(address)
if not toNickname:
continue
toDomain,toPort=getDomainFromActor(address)
if not toDomain:
continue
2019-07-15 18:29:30 +00:00
if debug:
2019-07-16 10:19:04 +00:00
domainFull=domain
if port:
if port!=80 and port!=443:
domainFull=domain+':'+str(port)
toDomainFull=toDomain
if toPort:
if toPort!=80 and toPort!=443:
toDomainFull=toDomain+':'+str(toPort)
print('Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
cc=[]
2019-07-15 18:20:52 +00:00
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
toNickname,toDomain,toPort, \
cc,httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug)
def sendToFollowers(session,baseDir: str, \
nickname: str, domain: str, port: int, \
httpPrefix: str,federationList: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
postJsonObject: {},debug: bool) -> None:
2019-07-08 13:30:04 +00:00
"""sends a post to the followers of the given nickname
"""
2019-07-16 10:19:04 +00:00
if not session:
print('WARN: No session for sendToFollowers')
return
2019-07-08 13:30:04 +00:00
if not postIsAddressedToFollowers(baseDir,nickname,domain, \
port,httpPrefix,postJsonObject):
2019-07-15 18:29:30 +00:00
if debug:
print('Post is not addressed to followers')
2019-07-08 13:30:04 +00:00
return
grouped=groupFollowersByDomain(baseDir,nickname,domain)
if not grouped:
2019-07-15 18:29:30 +00:00
if debug:
print('Post to followers did not resolve any domains')
2019-07-08 13:30:04 +00:00
return
2019-07-15 18:29:30 +00:00
# this is after the message has arrived at the server
2019-07-15 18:20:52 +00:00
clientToServer=False
2019-07-08 13:30:04 +00:00
# for each instance
for followerDomain,followerHandles in grouped.items():
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: follower handles for '+followerDomain)
pprint(followerHandles)
2019-07-08 13:30:04 +00:00
toPort=port
index=0
toDomain=followerHandles[index].split('@')[1]
if ':' in toDomain:
toPort=toDomain.split(':')[1]
toDomain=toDomain.split(':')[0]
toNickname=followerHandles[index].split('@')[0]
cc=''
if len(followerHandles)>1:
nickname='inbox'
toNickname='inbox'
2019-07-15 18:29:30 +00:00
if debug:
2019-07-16 22:57:45 +00:00
print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
2019-07-08 13:30:04 +00:00
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
toNickname,toDomain,toPort, \
cc,httpPrefix,True,clientToServer, \
2019-07-09 14:20:23 +00:00
federationList, \
2019-07-08 13:30:04 +00:00
sendThreads,postLog,cachedWebfingers, \
personCache,debug)
2019-07-16 22:57:45 +00:00
if debug:
print('DEBUG: End of sendToFollowers')
2019-07-08 13:30:04 +00:00
2019-07-04 16:24:23 +00:00
def createInbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
2019-07-04 16:24:23 +00:00
return createBoxBase(baseDir,'inbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
2019-07-03 19:00:03 +00:00
def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
2019-07-04 16:24:23 +00:00
return createBoxBase(baseDir,'outbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,authorized,False,pageNumber)
2019-07-04 16:24:23 +00:00
def getStatusNumberFromPostFilename(filename) -> int:
"""Gets the status number from a post filename
eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
returns 156295850695206
"""
if '#statuses#' not in filename:
return None
return int(filename.split('#')[-1].replace('.json',''))
2019-07-06 17:00:22 +00:00
def createBoxBase(baseDir: str,boxname: str, \
nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized :bool, \
ocapAlways: bool,pageNumber=None) -> {}:
"""Constructs the box feed for a person with the given nickname
2019-06-29 13:17:02 +00:00
"""
2019-07-04 16:24:23 +00:00
if boxname!='inbox' and boxname!='outbox':
return None
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
sharedBoxDir=None
if boxname=='inbox':
sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
2019-06-30 19:01:43 +00:00
if port!=80 and port!=443:
domain = domain+':'+str(port)
2019-06-29 16:47:37 +00:00
pageStr='?page=true'
if pageNumber:
try:
pageStr='?page='+str(pageNumber)
except:
pass
2019-07-04 16:24:23 +00:00
boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'totalItems': 0,
'type': 'OrderedCollection'}
boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
'orderedItems': [
],
'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'type': 'OrderedCollectionPage'}
2019-06-29 13:17:02 +00:00
# counter for posts loop
2019-06-29 16:47:37 +00:00
postsOnPageCtr=0
2019-06-29 13:17:02 +00:00
# post filenames sorted in descending order
postsInBoxDict={}
postsCtr=0
postsInPersonInbox=os.listdir(boxDir)
for postFilename in postsInPersonInbox:
# extract the status number
statusNumber=getStatusNumberFromPostFilename(postFilename)
if statusNumber:
postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
postsCtr+=1
# combine the inbox for the account with the shared inbox
if sharedBoxDir:
handle=nickname+'@'+domain
followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
postsInSharedInbox=os.listdir(sharedBoxDir)
for postFilename in postsInSharedInbox:
statusNumber=getStatusNumberFromPostFilename(postFilename)
if statusNumber:
sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
# get the actor from the shared post
with open(sharedInboxFilename, 'r') as fp:
2019-07-14 16:57:06 +00:00
postJsonObject=commentjson.load(fp)
actorNickname=getNicknameFromActor(postJsonObject['actor'])
actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
if actorNickname and actorDomain:
# is the actor followed by this account?
if actorNickname+'@'+actorDomain in open(followingFilename).read():
if ocapAlways:
capsList=None
# Note: should this be in the Create or the object of a post?
2019-07-14 16:57:06 +00:00
if postJsonObject.get('capability'):
if isinstance(postJsonObject['capability'], list):
capsList=postJsonObject['capability']
# Have capabilities been granted for the sender?
2019-07-14 16:57:06 +00:00
ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
if os.path.isfile(ocapFilename):
# read the capabilities id
with open(ocapFilename, 'r') as fp:
ocapJson=commentjson.load(fp)
if ocapJson.get('id'):
if ocapJson['id'] in capsList:
postsInBoxDict[statusNumber]=sharedInboxFilename
postsCtr+=1
else:
postsInBoxDict[statusNumber]=sharedInboxFilename
postsCtr+=1
# sort the list in descending order of date
postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=True))
2019-06-29 13:17:02 +00:00
2019-07-04 16:24:23 +00:00
# number of posts in box
boxHeader['totalItems']=postsCtr
2019-06-29 13:17:02 +00:00
prevPostFilename=None
2019-06-29 17:07:43 +00:00
if not pageNumber:
pageNumber=1
2019-06-29 13:17:02 +00:00
# Generate first and last entries within header
if postsCtr>0:
lastPage=int(postsCtr/itemsPerPage)
2019-06-29 17:07:43 +00:00
if lastPage<1:
lastPage=1
2019-07-04 16:24:23 +00:00
boxHeader['last']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
2019-06-29 13:17:02 +00:00
# Insert posts
2019-06-29 16:47:37 +00:00
currPage=1
postsCtr=0
for statusNumber,postFilename in postsInBox.items():
2019-06-29 16:47:37 +00:00
# Are we at the starting page yet?
if prevPostFilename and currPage==pageNumber and postsCtr==0:
# update the prev entry for the last message id
postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
2019-07-04 16:24:23 +00:00
boxHeader['prev']= \
2019-07-06 17:00:22 +00:00
httpPrefix+'://'+domain+'/users/'+nickname+'/'+ \
boxname+'?min_id='+postId+'&page=true'
2019-06-29 13:23:46 +00:00
# get the full path of the post file
filePath = postFilename
2019-06-29 13:17:02 +00:00
try:
if os.path.isfile(filePath):
2019-06-29 16:47:37 +00:00
if currPage == pageNumber and postsOnPageCtr <= itemsPerPage:
2019-06-29 13:23:46 +00:00
# get the post as json
2019-06-29 13:17:02 +00:00
with open(filePath, 'r') as fp:
p=commentjson.load(fp)
2019-07-12 12:43:18 +00:00
# Don't show likes or replies to unauthorized viewers
if not authorized:
2019-07-12 12:43:18 +00:00
if p.get('object'):
if isinstance(p['object'], dict):
if p['object'].get('likes'):
p['likes']={}
if p['object'].get('replies'):
p['replies']={}
2019-07-04 16:24:23 +00:00
# insert it into the box feed
2019-06-29 16:47:37 +00:00
if postsOnPageCtr < itemsPerPage:
if not headerOnly:
2019-07-04 16:24:23 +00:00
boxItems['orderedItems'].append(p)
2019-06-29 16:47:37 +00:00
elif postsOnPageCtr == itemsPerPage:
2019-06-29 13:23:46 +00:00
# if this is the last post update the next message ID
2019-06-29 13:17:02 +00:00
if '/statuses/' in p['id']:
postId = p['id'].split('/statuses/')[1].replace('/activity','')
2019-07-04 16:24:23 +00:00
boxHeader['next']= \
2019-07-03 19:00:03 +00:00
httpPrefix+'://'+domain+'/users/'+ \
2019-07-04 16:24:23 +00:00
nickname+'/'+boxname+'?max_id='+ \
2019-06-29 13:17:02 +00:00
postId+'&page=true'
2019-06-29 16:47:37 +00:00
postsOnPageCtr += 1
2019-06-29 13:23:46 +00:00
# remember the last post filename for use with prev
2019-06-29 13:17:02 +00:00
prevPostFilename = postFilename
2019-06-29 16:47:37 +00:00
if postsOnPageCtr > itemsPerPage:
2019-06-29 13:17:02 +00:00
break
2019-06-29 16:47:37 +00:00
# count the pages
postsCtr += 1
if postsCtr >= itemsPerPage:
postsCtr = 0
currPage += 1
2019-06-29 13:17:02 +00:00
except Exception as e:
print(e)
2019-06-29 16:47:37 +00:00
if headerOnly:
2019-07-04 16:24:23 +00:00
return boxHeader
return boxItems
2019-06-29 13:44:21 +00:00
2019-07-14 17:02:41 +00:00
def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str,maxPostsInBox=256) -> None:
2019-07-12 20:43:55 +00:00
"""Archives posts for all accounts
"""
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts'):
os.mkdir(archiveDir+'/accounts')
for subdir, dirs, files in os.walk(baseDir+'/accounts'):
for handle in dirs:
if '@' in handle:
nickname=handle.split('@')[0]
domain=handle.split('@')[1]
archiveSubdir=None
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts/'+handle):
os.mkdir(archiveDir+'/accounts/'+handle)
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
2019-07-14 17:02:41 +00:00
archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
2019-07-12 20:43:55 +00:00
'inbox',archiveSubdir, \
maxPostsInBox)
if archiveDir:
archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
2019-07-14 17:02:41 +00:00
archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
2019-07-12 20:43:55 +00:00
'outbox',archiveSubdir, \
maxPostsInBox)
2019-07-14 17:02:41 +00:00
def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
2019-07-12 20:43:55 +00:00
boxname: str,archiveDir: str,maxPostsInBox=256) -> None:
2019-07-04 16:24:23 +00:00
"""Retain a maximum number of posts within the given box
2019-06-29 13:44:21 +00:00
Move any others to an archive directory
"""
2019-07-04 16:24:23 +00:00
if boxname!='inbox' and boxname!='outbox':
return
2019-07-12 20:43:55 +00:00
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
2019-07-04 16:24:23 +00:00
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
postsInBox=sorted(os.listdir(boxDir), reverse=False)
noOfPosts=len(postsInBox)
if noOfPosts<=maxPostsInBox:
2019-06-29 13:44:21 +00:00
return
2019-07-04 16:24:23 +00:00
for postFilename in postsInBox:
2019-07-14 15:43:02 +00:00
filePath = os.path.join(boxDir, postFilename)
2019-06-29 13:44:21 +00:00
if os.path.isfile(filePath):
2019-07-12 20:43:55 +00:00
if archiveDir:
2019-07-14 16:37:01 +00:00
repliesPath=filePath.replace('.json','.replies')
2019-07-12 20:43:55 +00:00
archivePath = os.path.join(archiveDir, postFilename)
os.rename(filePath,archivePath)
2019-07-14 15:43:02 +00:00
if os.path.isfile(repliesPath):
os.rename(repliesPath,archivePath)
2019-07-12 20:43:55 +00:00
else:
2019-07-14 17:02:41 +00:00
deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
2019-06-29 13:44:21 +00:00
noOfPosts -= 1
2019-07-04 16:24:23 +00:00
if noOfPosts <= maxPostsInBox:
2019-06-29 13:44:21 +00:00
break
2019-07-03 10:31:02 +00:00
2019-07-06 17:00:22 +00:00
def getPublicPostsOfPerson(nickname: str,domain: str, \
raw: bool,simple: bool) -> None:
2019-07-03 10:31:02 +00:00
""" This is really just for test purposes
"""
useTor=True
2019-07-03 10:33:55 +00:00
port=443
2019-07-03 10:31:02 +00:00
session = createSession(domain,port,useTor)
personCache={}
cachedWebfingers={}
federationList=[]
2019-07-04 17:31:41 +00:00
httpPrefix='https'
handle=httpPrefix+"://"+domain+"/@"+nickname
2019-07-06 17:00:22 +00:00
wfRequest = \
webfingerHandle(session,handle,httpPrefix,cachedWebfingers)
2019-07-03 10:31:02 +00:00
if not wfRequest:
sys.exit()
2019-07-05 20:32:21 +00:00
personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition= \
2019-07-05 13:50:27 +00:00
getPersonBox(session,wfRequest,personCache,'outbox')
2019-07-03 10:31:02 +00:00
wfResult = json.dumps(wfRequest, indent=4, sort_keys=True)
maxMentions=10
maxEmoji=10
maxAttachments=5
2019-07-06 17:00:22 +00:00
userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
2019-07-09 14:20:23 +00:00
maxAttachments,federationList, \
2019-07-06 21:58:56 +00:00
personCache,raw,simple)
2019-07-03 10:31:02 +00:00
#print(str(userPosts))
2019-07-09 14:20:23 +00:00
def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
nickname: str,domain: str,port: int, \
followerUrl,updateCaps: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
federationList :[],debug :bool) -> int:
"""When the capabilities for a follower are changed this
sends out an update. followerUrl is the actor of the follower.
"""
updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
nickname,domain,port, \
followerUrl, \
updateCaps)
if not updateJson:
return 1
if debug:
pprint(updateJson)
print('DEBUG: sending capabilities update from '+ \
nickname+'@'+domain+' port '+ str(port) + \
' to '+followerUrl)
clientToServer=False
followerNickname=getNicknameFromActor(followerUrl)
followerDomain,followerPort=getDomainFromActor(followerUrl)
return sendSignedJson(updateJson,session,baseDir, \
nickname,domain,port, \
followerNickname,followerDomain,followerPort, '', \
httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug)