mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
epicyon/posts.py

2481 lines
100 KiB
Python
Raw Blame History

__filename__ = "posts.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.0.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
import requests
import json
import commentjson
import html
import datetime
import os
import shutil
import threading
import sys
import trace
import time
from time import gmtime, strftime
from collections import OrderedDict
from threads import threadWithTrace
from cache import storePersonInCache
from cache import getPersonFromCache
from cache import expirePersonCache
from pprint import pprint
from random import randint
from session import createSession
from session import getJson
from session import postJsonString
from session import postImage
from webfinger import webfingerHandle
from httpsig import createSignedHeader
from utils import getStatusNumber
from utils import createPersonDir
from utils import urlPermitted
from utils import getNicknameFromActor
from utils import getDomainFromActor
from utils import deletePost
from utils import validNickname
from utils import locatePost
from capabilities import getOcapFilename
from capabilities import capabilitiesUpdate
from media import attachMedia
from content import addHtmlTags
from content import replaceEmojiFromTags
from auth import createBasicAuthHeader
from config import getConfigParam
from blocking import isBlocked
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
from bs4 import BeautifulSoup
def isModerator(baseDir: str,nickname: str) -> bool:
"""Returns true if the given nickname is a moderator
"""
moderatorsFile=baseDir+'/accounts/moderators.txt'
if not os.path.isfile(moderatorsFile):
if getConfigParam(baseDir,'admin')==nickname:
return True
return False
with open(moderatorsFile, "r") as f:
lines = f.readlines()
if len(lines)==0:
if getConfigParam(baseDir,'admin')==nickname:
return True
for moderator in lines:
moderator=moderator.strip('\n')
if moderator==nickname:
return True
return False
def noOfFollowersOnDomain(baseDir: str,handle: str, \
domain: str, followFile='followers.txt') -> int:
"""Returns the number of followers of the given handle from the given domain
"""
filename=baseDir+'/accounts/'+handle+'/'+followFile
if not os.path.isfile(filename):
return 0
ctr=0
with open(filename, "r") as followersFilename:
for followerHandle in followersFilename:
if '@' in followerHandle:
followerDomain= \
followerHandle.split('@')[1].replace('\n','')
if domain==followerDomain:
ctr+=1
return ctr
def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
debug=False):
"""Returns the public or private key of a person
"""
handle=nickname+'@'+domain
keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
if not os.path.isfile(keyFilename):
if debug:
print('DEBUG: private key file not found: '+keyFilename)
return ''
keyPem=''
with open(keyFilename, "r") as pemFile:
keyPem=pemFile.read()
if len(keyPem)<20:
if debug:
print('DEBUG: private key was too short: '+keyPem)
return ''
return keyPem
def cleanHtml(rawHtml: str) -> str:
text = BeautifulSoup(rawHtml, 'html.parser').get_text()
return html.unescape(text)
def getUserUrl(wfRequest) -> str:
if wfRequest.get('links'):
for link in wfRequest['links']:
if link.get('type') and link.get('href'):
if link['type'] == 'application/activity+json':
if '/users/' in link['href'] or \
'/profile/' in link['href'] or \
'/channel/' in link['href']:
return link['href']
else:
print('Webfinger activity+json does not contain a valid actor')
print(link['href'])
return None
def parseUserFeed(session,feedUrl: str,asHeader: {}, \
projectVersion: str,httpPrefix: str,domain: str) -> None:
feedJson = getJson(session,feedUrl,asHeader,None, \
projectVersion,httpPrefix,domain)
if not feedJson:
return
if 'orderedItems' in feedJson:
for item in feedJson['orderedItems']:
yield item
nextUrl = None
if 'first' in feedJson:
nextUrl = feedJson['first']
elif 'next' in feedJson:
nextUrl = feedJson['next']
if nextUrl:
if isinstance(nextUrl, str):
userFeed=parseUserFeed(session,nextUrl,asHeader, \
projectVersion,httpPrefix,domain)
for item in userFeed:
yield item
elif isinstance(nextUrl, dict):
userFeed=nextUrl
if userFeed.get('orderedItems'):
for item in userFeed['orderedItems']:
yield item
def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
projectVersion: str,httpPrefix: str, \
nickname: str,domain: str, \
boxName='inbox') -> (str,str,str,str,str,str,str,str):
asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
if not wfRequest.get('errors'):
personUrl = getUserUrl(wfRequest)
else:
personUrl = httpPrefix+'://'+domain+'/users/'+nickname
if not personUrl:
return None,None,None,None,None,None,None,None
personJson = getPersonFromCache(baseDir,personUrl,personCache)
if not personJson:
if '/channel/' in personUrl:
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
personJson = getJson(session,personUrl,asHeader,None, \
projectVersion,httpPrefix,domain)
if not personJson:
return None,None,None,None,None,None,None,None
boxJson=None
if not personJson.get(boxName):
if personJson.get('endpoints'):
if personJson['endpoints'].get(boxName):
boxJson=personJson['endpoints'][boxName]
else:
boxJson=personJson[boxName]
if not boxJson:
return None,None,None,None,None,None,None,None
personId=None
if personJson.get('id'):
personId=personJson['id']
pubKeyId=None
pubKey=None
if personJson.get('publicKey'):
if personJson['publicKey'].get('id'):
pubKeyId=personJson['publicKey']['id']
if personJson['publicKey'].get('publicKeyPem'):
pubKey=personJson['publicKey']['publicKeyPem']
sharedInbox=None
if personJson.get('sharedInbox'):
sharedInbox=personJson['sharedInbox']
else:
if personJson.get('endpoints'):
if personJson['endpoints'].get('sharedInbox'):
sharedInbox=personJson['endpoints']['sharedInbox']
capabilityAcquisition=None
if personJson.get('capabilityAcquisitionEndpoint'):
capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
avatarUrl=None
if personJson.get('icon'):
if personJson['icon'].get('url'):
avatarUrl=personJson['icon']['url']
displayName=None
if personJson.get('name'):
displayName=personJson['name']
storePersonInCache(baseDir,personUrl,personJson,personCache)
return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
def getPosts(session,outboxUrl: str,maxPosts: int, \
maxMentions: int, \
maxEmoji: int,maxAttachments: int, \
federationList: [], \
personCache: {},raw: bool, \
simple: bool,debug: bool, \
projectVersion: str,httpPrefix: str,domain: str) -> {}:
"""Gets public posts from an outbox
"""
personPosts={}
if not outboxUrl:
return personPosts
asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
if '/outbox/' in outboxUrl:
asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
if raw:
result = []
i = 0
userFeed=parseUserFeed(session,outboxUrl,asHeader, \
projectVersion,httpPrefix,domain)
for item in userFeed:
result.append(item)
i += 1
if i == maxPosts:
break
pprint(result)
return None
i = 0
userFeed=parseUserFeed(session,outboxUrl,asHeader, \
projectVersion,httpPrefix,domain)
for item in userFeed:
if not item.get('id'):
if debug:
print('No id')
continue
if not item.get('type'):
if debug:
print('No type')
continue
if item['type'] != 'Create':
if debug:
print('Not Create type')
continue
if not item.get('object'):
if debug:
print('No object')
continue
if not isinstance(item['object'], dict):
if debug:
print('item object is not a dict')
continue
if not item['object'].get('published'):
if debug:
print('No published attribute')
continue
#pprint(item)
published = item['object']['published']
if not personPosts.get(item['id']):
# check that this is a public post
# #Public should appear in the "to" list
if item['object'].get('to'):
isPublic=False
for recipient in item['object']['to']:
if recipient.endswith('#Public'):
isPublic=True
break
if not isPublic:
continue
content = item['object']['content'].replace('&apos;',"'")
mentions=[]
emoji={}
if item['object'].get('tag'):
for tagItem in item['object']['tag']:
tagType=tagItem['type'].lower()
if tagType=='emoji':
if tagItem.get('name') and tagItem.get('icon'):
if tagItem['icon'].get('url'):
# No emoji from non-permitted domains
if urlPermitted(tagItem['icon']['url'], \
federationList, \
"objects:read"):
emojiName=tagItem['name']
emojiIcon=tagItem['icon']['url']
emoji[emojiName]=emojiIcon
else:
if debug:
print('url not permitted '+tagItem['icon']['url'])
if tagType=='mention':
if tagItem.get('name'):
if tagItem['name'] not in mentions:
mentions.append(tagItem['name'])
if len(mentions)>maxMentions:
if debug:
print('max mentions reached')
continue
if len(emoji)>maxEmoji:
if debug:
print('max emojis reached')
continue
summary = ''
if item['object'].get('summary'):
if item['object']['summary']:
summary = item['object']['summary']
inReplyTo = ''
if item['object'].get('inReplyTo'):
if item['object']['inReplyTo']:
# No replies to non-permitted domains
if not urlPermitted(item['object']['inReplyTo'], \
federationList, \
"objects:read"):
if debug:
print('url not permitted '+item['object']['inReplyTo'])
continue
inReplyTo = item['object']['inReplyTo']
conversation = ''
if item['object'].get('conversation'):
if item['object']['conversation']:
# no conversations originated in non-permitted domains
if urlPermitted(item['object']['conversation'], \
federationList,"objects:read"):
conversation = item['object']['conversation']
attachment = []
if item['object'].get('attachment'):
if item['object']['attachment']:
for attach in item['object']['attachment']:
if attach.get('name') and attach.get('url'):
# no attachments from non-permitted domains
if urlPermitted(attach['url'], \
federationList, \
"objects:read"):
attachment.append([attach['name'],attach['url']])
else:
if debug:
print('url not permitted '+attach['url'])
sensitive = False
if item['object'].get('sensitive'):
sensitive = item['object']['sensitive']
if simple:
print(cleanHtml(content)+'\n')
else:
pprint(item)
personPosts[item['id']] = {
"sensitive": sensitive,
"inreplyto": inReplyTo,
"summary": summary,
"html": content,
"plaintext": cleanHtml(content),
"attachment": attachment,
"mentions": mentions,
"emoji": emoji,
"conversation": conversation
}
i += 1
if i == maxPosts:
break
return personPosts
def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
"""Deletes all posts for a person from inbox or outbox
"""
if boxname!='inbox' and boxname!='outbox':
return
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
for deleteFilename in os.scandir(boxDir):
deleteFilename=deleteFilename.name
filePath = os.path.join(boxDir, deleteFilename)
try:
if os.path.isfile(filePath):
os.unlink(filePath)
elif os.path.isdir(filePath): shutil.rmtree(filePath)
except Exception as e:
print(e)
def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
nickname: str, domain: str,postJsonObject: {}, \
boxname: str) -> str:
"""Saves the give json to the give box
Returns the filename
"""
if boxname!='inbox' and boxname!='outbox':
return None
originalDomain=domain
if ':' in domain:
domain=domain.split(':')[0]
if not postId:
statusNumber,published = getStatusNumber()
postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
postJsonObject['id']=postId+'/activity'
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
postJsonObject['object']['id']=postId
postJsonObject['object']['atomUri']=postId
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
filename=boxDir+'/'+postId.replace('/','#')+'.json'
tries=0
while tries<5:
try:
with open(filename, 'w') as fp:
commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False)
break
except Exception as e:
print(e)
time.sleep(1)
tries+=1
return filename
def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
"""Writes the post url for hashtags to a file
This allows posts for a hashtag to be quickly looked up
"""
if tag['type']!='Hashtag':
return
# create hashtags directory
tagsDir=baseDir+'/tags'
if not os.path.isdir(tagsDir):
os.mkdir(tagsDir)
tagName=tag['name']
tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
tagFile=open(tagsFilename, "a+")
if not tagFile:
return
tagFile.write(newPostId+'\n')
tagFile.close()
def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
toUrl: str, ccUrl: str, httpPrefix: str, content: str, \
followersOnly: bool, saveToFile: bool, clientToServer: bool, \
attachImageFilename: str,mediaType: str,imageDescription: str, \
useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
inReplyToAtomUri=None, subject=None, \
eventDate=None,eventTime=None,location=None) -> {}:
"""Creates a message
"""
mentionedRecipients= \
getMentionedPeople(baseDir,httpPrefix,content,domain,False)
tags=[]
hashtagsDict={}
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
# convert content to html
emojisDict={}
content= \
addHtmlTags(baseDir,httpPrefix, \
nickname,domain,content, \
mentionedRecipients, \
hashtagsDict)
statusNumber,published = getStatusNumber()
postTo='https://www.w3.org/ns/activitystreams#Public'
postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
if followersOnly:
postTo=postCC
postCC=''
newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
sensitive=False
summary=None
if subject:
summary=subject
sensitive=True
toRecipients=[]
if toUrl:
if not isinstance(toUrl, str):
print('ERROR: toUrl is not a string')
return None
toRecipients=[toUrl]
# who to send to
if mentionedRecipients:
for mention in mentionedRecipients:
if mention not in toRecipients:
toRecipients.append(mention)
# create a list of hashtags
# Only posts which are #Public are searchable by hashtag
if hashtagsDict:
isPublic=False
for recipient in toRecipients:
if recipient.endswith('#Public'):
isPublic=True
break
for tagName,tag in hashtagsDict.items():
tags.append(tag)
if isPublic:
updateHashtagsIndex(baseDir,tag,newPostId)
print('Content tags: '+str(tags))
content=replaceEmojiFromTags(content,tags,'content')
if inReplyTo and not sensitive:
# locate the post which this is a reply to and check if
# it has a content warning. If it does then reproduce
# the same warning
replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo)
if replyPostFilename:
replyToJson=None
tries=0
while tries<5:
try:
with open(replyPostFilename, 'r') as fp:
replyToJson=commentjson.load(fp)
break
except Exception as e:
print('WARN: commentjson exception createPostBase - '+str(e))
time.sleep(1)
tries+=1
if replyToJson:
if replyToJson.get('object'):
if replyToJson['object'].get('sensitive'):
if replyToJson['object']['sensitive']:
sensitive=True
if replyToJson['object'].get('summary'):
summary=replyToJson['object']['summary']
if eventDate:
eventName=summary
if not eventName:
eventName=content
eventDateStr=eventDate
if eventTime:
if eventTime.endswith('Z'):
eventDateStr=eventDate+'T'+eventTime
else:
eventDateStr=eventDate+'T'+eventTime+':00'+strftime("%z", gmtime())
else:
eventDateStr=eventDate+'T12:00:00Z'
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Event",
"name": eventName,
"startTime": eventDateStr,
"endTime": eventDateStr
})
if location:
tags.append({
"@context": "https://www.w3.org/ns/activitystreams",
"type": "Place",
"name": location
})
if not clientToServer:
actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
# if capabilities have been granted for this actor
# then get the corresponding id
capabilityId=None
capabilityIdList=[]
ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
if ocapFilename:
if os.path.isfile(ocapFilename):
tries=0
while tries<5:
try:
with open(ocapFilename, 'r') as fp:
oc=commentjson.load(fp)
if oc.get('id'):
capabilityIdList=[oc['id']]
break
except Exception as e:
print('WARN: commentjson exception createPostBase - '+str(e))
time.sleep(1)
tries+=1
newPost = {
"@context": "https://www.w3.org/ns/activitystreams",
'id': newPostId+'/activity',
'capability': capabilityIdList,
'type': 'Create',
'actor': actorUrl,
'published': published,
'to': [toUrl],
'cc': [],
'object': {
'id': newPostId,
'type': 'Note',
'summary': summary,
'inReplyTo': inReplyTo,
'published': published,
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
'to': toRecipients,
'cc': [],
'sensitive': sensitive,
'atomUri': newPostId,
'inReplyToAtomUri': inReplyToAtomUri,
'content': content,
'contentMap': {
'en': content
},
'attachment': [],
'tag': tags,
'replies': {
'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'items': []
}
}
}
}
if attachImageFilename:
newPost['object']= \
attachMedia(baseDir,httpPrefix,domain,port, \
newPost['object'],attachImageFilename, \
mediaType,imageDescription,useBlurhash)
else:
newPost = {
"@context": "https://www.w3.org/ns/activitystreams",
'id': newPostId,
'type': 'Note',
'summary': summary,
'inReplyTo': inReplyTo,
'published': published,
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
'to': toRecipients,
'cc': [],
'sensitive': sensitive,
'atomUri': newPostId,
'inReplyToAtomUri': inReplyToAtomUri,
'content': content,
'contentMap': {
'en': content
},
'attachment': [],
'tag': tags,
'replies': {
'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'type': 'Collection',
'first': {
'type': 'CollectionPage',
'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
'items': []
}
}
}
if attachImageFilename:
newPost= \
attachMedia(baseDir,httpPrefix,domain,port, \
newPost,attachImageFilename, \
mediaType,imageDescription,useBlurhash)
if ccUrl:
if len(ccUrl)>0:
newPost['cc']=[ccUrl]
if newPost.get('object'):
newPost['object']['cc']=[ccUrl]
# if this is a moderation report then add a status
if isModerationReport:
# add status
if newPost.get('object'):
newPost['object']['moderationStatus']='pending'
else:
newPost['moderationStatus']='pending'
# save to index file
moderationIndexFile=baseDir+'/accounts/moderation.txt'
modFile=open(moderationIndexFile, "a+")
if modFile:
modFile.write(newPostId+'\n')
modFile.close()
if saveToFile:
savePostToBox(baseDir,httpPrefix,newPostId, \
nickname,domain,newPost,'outbox')
return newPost
def outboxMessageCreateWrap(httpPrefix: str, \
nickname: str,domain: str,port: int, \
messageJson: {}) -> {}:
"""Wraps a received message in a Create
https://www.w3.org/TR/activitypub/#object-without-create
"""
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
statusNumber,published = getStatusNumber()
if messageJson.get('published'):
published = messageJson['published']
newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
cc=[]
if messageJson.get('cc'):
cc=messageJson['cc']
# TODO
capabilityUrl=[]
newPost = {
"@context": "https://www.w3.org/ns/activitystreams",
'id': newPostId+'/activity',
'capability': capabilityUrl,
'type': 'Create',
'actor': httpPrefix+'://'+domain+'/users/'+nickname,
'published': published,
'to': messageJson['to'],
'cc': cc,
'object': messageJson
}
newPost['object']['id']=newPost['id']
newPost['object']['url']= \
httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
newPost['object']['atomUri']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
return newPost
def postIsAddressedToFollowers(baseDir: str,
nickname: str,domain: str,port: int,httpPrefix: str,
postJsonObject: {}) -> bool:
"""Returns true if the given post is addressed to followers of the nickname
"""
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
if not postJsonObject.get('object'):
return False
toList=[]
ccList=[]
if postJsonObject['type']!='Update' and \
isinstance(postJsonObject['object'], dict):
if not postJsonObject['object'].get('to'):
return False
toList=postJsonObject['object']['to']
if postJsonObject['object'].get('cc'):
ccList=postJsonObject['object']['cc']
else:
if not postJsonObject.get('to'):
return False
toList=postJsonObject['to']
if postJsonObject.get('cc'):
ccList=postJsonObject['cc']
followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
# does the followers url exist in 'to' or 'cc' lists?
addressedToFollowers=False
if followersUrl in toList:
addressedToFollowers=True
if not addressedToFollowers:
if followersUrl in ccList:
addressedToFollowers=True
return addressedToFollowers
def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
"""Returns true if the given post is addressed to public
"""
if not postJsonObject.get('object'):
return False
if not postJsonObject['object'].get('to'):
return False
publicUrl='https://www.w3.org/ns/activitystreams#Public'
# does the public url exist in 'to' or 'cc' lists?
addressedToPublic=False
if publicUrl in postJsonObject['object']['to']:
addressedToPublic=True
if not addressedToPublic:
if not postJsonObject['object'].get('cc'):
return False
if publicUrl in postJsonObject['object']['cc']:
addressedToPublic=True
return addressedToPublic
def createPublicPost(baseDir: str,
nickname: str,domain: str,port: int,httpPrefix: str, \
content: str,followersOnly: bool,saveToFile: bool,
clientToServer: bool,\
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
inReplyTo=None,inReplyToAtomUri=None,subject=None, \
eventDate=None,eventTime=None,location=None) -> {}:
"""Public post
"""
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
return createPostBase(baseDir,nickname,domain,port, \
'https://www.w3.org/ns/activitystreams#Public', \
httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
httpPrefix,content,followersOnly,saveToFile, \
clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo,inReplyToAtomUri,subject, \
eventDate,eventTime,location)
def createUnlistedPost(baseDir: str,
nickname: str,domain: str,port: int,httpPrefix: str, \
content: str,followersOnly: bool,saveToFile: bool,
clientToServer: bool,\
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
inReplyTo=None,inReplyToAtomUri=None,subject=None, \
eventDate=None,eventTime=None,location=None) -> {}:
"""Unlisted post. This has the #Public and followers links inverted.
"""
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
return createPostBase(baseDir,nickname,domain,port, \
httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
'https://www.w3.org/ns/activitystreams#Public', \
httpPrefix,content,followersOnly,saveToFile, \
clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo, inReplyToAtomUri, subject, \
eventDate,eventTime,location)
def createFollowersOnlyPost(baseDir: str,
nickname: str,domain: str,port: int,httpPrefix: str, \
content: str,followersOnly: bool,saveToFile: bool,
clientToServer: bool,\
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
inReplyTo=None,inReplyToAtomUri=None,subject=None, \
eventDate=None,eventTime=None,location=None) -> {}:
"""Followers only post
"""
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
return createPostBase(baseDir,nickname,domain,port, \
httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
None,
httpPrefix,content,followersOnly,saveToFile, \
clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo, inReplyToAtomUri, subject, \
eventDate,eventTime,location)
def getMentionedPeople(baseDir: str,httpPrefix: str, \
content: str,domain: str,debug: bool) -> []:
"""Extracts a list of mentioned actors from the given message content
"""
if '@' not in content:
return None
mentions=[]
words=content.split(' ')
for wrd in words:
if wrd.startswith('@'):
handle=wrd[1:]
if debug:
print('DEBUG: mentioned handle '+handle)
if '@' not in handle:
handle=handle+'@'+domain
if not os.path.isdir(baseDir+'/accounts/'+handle):
continue
else:
externalDomain=handle.split('@')[1]
if not ('.' in externalDomain or externalDomain=='localhost'):
continue
mentionedNickname=handle.split('@')[0]
mentionedDomain=handle.split('@')[1].strip('\n')
if ':' in mentionedDomain:
mentionedDomain=mentionedDomain.split(':')[0]
if not validNickname(mentionedDomain,mentionedNickname):
continue
actor=httpPrefix+'://'+handle.split('@')[1]+'/users/'+mentionedNickname
mentions.append(actor)
return mentions
def createDirectMessagePost(baseDir: str,
nickname: str,domain: str,port: int,httpPrefix: str, \
content: str,followersOnly: bool,saveToFile: bool,
clientToServer: bool,\
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
inReplyTo=None,inReplyToAtomUri=None,subject=None,debug=False, \
eventDate=None,eventTime=None,location=None) -> {}:
"""Direct Message post
"""
mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
if debug:
print('mentionedPeople: '+str(mentionedPeople))
if not mentionedPeople:
return None
postTo=None
postCc=None
return createPostBase(baseDir,nickname,domain,port, \
postTo,postCc, \
httpPrefix,content,followersOnly,saveToFile, \
clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo,inReplyToAtomUri,subject, \
eventDate,eventTime,location)
def createReportPost(baseDir: str,
nickname: str, domain: str, port: int,httpPrefix: str, \
content: str, followersOnly: bool, saveToFile: bool,
clientToServer: bool,\
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
debug: bool,subject=None) -> {}:
"""Send a report to moderators
"""
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
# add a title to distinguish moderation reports from other posts
reportTitle='Moderation Report'
if not subject:
subject=reportTitle
else:
if not subject.startswith(reportTitle):
subject=reportTitle+': '+subject
# create the list of moderators from the moderators file
moderatorsList=[]
moderatorsFile=baseDir+'/accounts/moderators.txt'
if os.path.isfile(moderatorsFile):
with open (moderatorsFile, "r") as fileHandler:
for line in fileHandler:
line=line.strip('\n')
if line.startswith('#'):
continue
if line.startswith('/users/'):
line=line.replace('users','')
if line.startswith('@'):
line=line[1:]
if '@' in line:
moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
if moderatorActor not in moderatorList:
moderatorsList.append(moderatorActor)
continue
if line.startswith('http') or line.startswith('dat'):
# must be a local address - no remote moderators
if '://'+domainFull+'/' in line:
if line not in moderatorsList:
moderatorsList.append(line)
else:
if '/' not in line:
moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
if moderatorActor not in moderatorsList:
moderatorsList.append(moderatorActor)
if len(moderatorsList)==0:
# if there are no moderators then the admin becomes the moderator
adminNickname=getConfigParam(baseDir,'admin')
if adminNickname:
moderatorsList.append(httpPrefix+'://'+domainFull+'/users/'+adminNickname)
if not moderatorsList:
return None
if debug:
print('DEBUG: Sending report to moderators')
print(str(moderatorsList))
postTo=moderatorsList
postCc=None
postJsonObject=None
for toUrl in postTo:
postJsonObject= \
createPostBase(baseDir,nickname, domain, port, \
toUrl,postCc, \
httpPrefix, content, followersOnly, saveToFile, \
clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
True,None, None, subject, \
None,None,None)
return postJsonObject
def threadSendPost(session,postJsonStr: str,federationList: [],\
inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
debug :bool) -> None:
"""Sends a post with exponential backoff
"""
tries=0
backoffTime=60
for attempt in range(20):
postResult=None
try:
postResult = \
postJsonString(session,postJsonStr,federationList, \
inboxUrl,signatureHeaderJson, \
"inbox:write",debug)
except Exception as e:
print('ERROR: postJsonString failed')
print(e)
if postResult:
logStr='Success on try '+str(tries)+': '+postJsonStr
else:
logStr='Retry '+str(tries)+': '+postJsonStr
postLog.append(logStr)
# keep the length of the log finite
# Don't accumulate massive files on systems with limited resources
while len(postLog)>16:
postLog.pop(0)
if debug:
# save the log file
postLogFilename=baseDir+'/post.log'
with open(postLogFilename, "a+") as logFile:
logFile.write(logStr+'\n')
if postResult:
if debug:
print('DEBUG: successful json post to '+inboxUrl)
# our work here is done
break
if debug:
print(postJsonStr)
print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
str(backoffTime)+' seconds.')
time.sleep(backoffTime)
backoffTime *= 2
tries+=1
def sendPost(projectVersion: str, \
session,baseDir: str,nickname: str, domain: str, port: int, \
toNickname: str, toDomain: str, toPort: int, cc: str, \
httpPrefix: str, content: str, followersOnly: bool, \
saveToFile: bool, clientToServer: bool, \
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
federationList: [],\
sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
"""Post to another inbox
"""
withDigest=True
if toNickname=='inbox':
# shared inbox actor on @domain@domain
toNickname=toDomain
toDomainOriginal=toDomain
if toPort:
if toPort!=80 and toPort!=443:
if ':' not in toDomain:
toDomain=toDomain+':'+str(toPort)
handle=httpPrefix+'://'+toDomain+'/@'+toNickname
# lookup the inbox for the To handle
wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
domain,projectVersion)
if not wfRequest:
return 1
if not clientToServer:
postToBox='inbox'
else:
postToBox='outbox'
# get the actor inbox for the To handle
inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
getPersonBox(baseDir,session,wfRequest,personCache, \
projectVersion,httpPrefix, \
nickname,domain,postToBox)
# If there are more than one followers on the target domain
# then send to the shared inbox indead of the individual inbox
if nickname=='capabilities':
inboxUrl=capabilityAcquisition
if not capabilityAcquisition:
return 2
if not inboxUrl:
return 3
if not pubKey:
return 4
if not toPersonId:
return 5
# sharedInbox and capabilities are optional
postJsonObject = \
createPostBase(baseDir,nickname,domain,port, \
toPersonId,cc,httpPrefix,content, \
followersOnly,saveToFile,clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo,inReplyToAtomUri,subject, \
None,None,None)
# get the senders private key
privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
if len(privateKeyPem)==0:
return 6
if toDomain not in inboxUrl:
return 7
postPath=inboxUrl.split(toDomain,1)[1]
# convert json to string so that there are no
# subsequent conversions after creating message body digest
postJsonStr=json.dumps(postJsonObject)
# construct the http header, including the message body digest
signatureHeaderJson = \
createSignedHeader(privateKeyPem,nickname,domain,port, \
toDomain,toPort, \
postPath,httpPrefix,withDigest,postJsonStr)
# Keep the number of threads being used small
while len(sendThreads)>20:
print('WARN: Maximum threads reached - killing send thread')
sendThreads[0].kill()
sendThreads.pop(0)
print('WARN: thread killed')
thr = threadWithTrace(target=threadSendPost,args=(session, \
postJsonStr, \
federationList, \
inboxUrl,baseDir, \
signatureHeaderJson.copy(), \
postLog,
debug),daemon=True)
sendThreads.append(thr)
thr.start()
return 0
def sendPostViaServer(projectVersion: str, \
baseDir: str,session,fromNickname: str,password: str, \
fromDomain: str, fromPort: int, \
toNickname: str, toDomain: str, toPort: int, cc: str, \
httpPrefix: str, content: str, followersOnly: bool, \
attachImageFilename: str,mediaType: str, \
imageDescription: str,useBlurhash: bool, \
cachedWebfingers: {},personCache: {}, \
debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
"""Send a post via a proxy (c2s)
"""
if not session:
print('WARN: No session for sendPostViaServer')
return 6
withDigest=True
if toPort:
if toPort!=80 and toPort!=443:
if ':' not in fromDomain:
fromDomain=fromDomain+':'+str(fromPort)
handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
# lookup the inbox for the To handle
wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
fromDomain,projectVersion)
if not wfRequest:
if debug:
print('DEBUG: webfinger failed for '+handle)
return 1
postToBox='outbox'
# get the actor inbox for the To handle
inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
getPersonBox(baseDir,session,wfRequest,personCache, \
projectVersion,httpPrefix,fromNickname, \
fromDomain,postToBox)
if not inboxUrl:
if debug:
print('DEBUG: No '+postToBox+' was found for '+handle)
return 3
if not fromPersonId:
if debug:
print('DEBUG: No actor was found for '+handle)
return 4
# Get the json for the c2s post, not saving anything to file
# Note that baseDir is set to None
saveToFile=False
clientToServer=True
if toDomain.lower().endswith('public'):
toPersonId='https://www.w3.org/ns/activitystreams#Public'
fromDomainFull=fromDomain
if fromPort:
if fromPort!=80 and fromPort!=443:
if ':' not in fromDomain:
fromDomainFull=fromDomain+':'+str(fromPort)
cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
else:
if toDomain.lower().endswith('followers') or \
toDomain.lower().endswith('followersonly'):
toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
else:
toDomainFull=toDomain
if toPort:
if toPort!=80 and toPort!=443:
if ':' not in toDomain:
toDomainFull=toDomain+':'+str(toPort)
toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
postJsonObject = \
createPostBase(baseDir, \
fromNickname,fromDomain,fromPort, \
toPersonId,cc,httpPrefix,content, \
followersOnly,saveToFile,clientToServer, \
attachImageFilename,mediaType, \
imageDescription,useBlurhash, \
False,inReplyTo,inReplyToAtomUri,subject, \
None,None,None)
authHeader=createBasicAuthHeader(fromNickname,password)
if attachImageFilename:
headers = {'host': fromDomain, \
'Authorization': authHeader}
postResult = \
postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
#if not postResult:
# if debug:
# print('DEBUG: Failed to upload image')
# return 9
headers = {'host': fromDomain, \
'Content-type': 'application/json', \
'Authorization': authHeader}
postResult = \
postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write",debug)
#if not postResult:
# if debug:
# print('DEBUG: POST failed for c2s to '+inboxUrl)
# return 5
if debug:
print('DEBUG: c2s POST success')
return 0
def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
"""Returns a dictionary with followers grouped by domain
"""
handle=nickname+'@'+domain
followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
if not os.path.isfile(followersFilename):
return None
grouped={}
with open(followersFilename, "r") as f:
for followerHandle in f:
if '@' in followerHandle:
fHandle=followerHandle.strip().replace('\n','')
followerDomain=fHandle.split('@')[1]
if not grouped.get(followerDomain):
grouped[followerDomain]=[fHandle]
else:
grouped[followerDomain].append(fHandle)
return grouped
def addFollowersToPublicPost(postJsonObject: {}) -> None:
"""Adds followers entry to cc if it doesn't exist
"""
if not postJsonObject.get('actor'):
return
if isinstance(postJsonObject['object'], str):
if not postJsonObject.get('to'):
return
if len(postJsonObject['to'])>1:
return
if len(postJsonObject['to'])==0:
return
if not postJsonObject['to'][0].endswith('#Public'):
return
if postJsonObject.get('cc'):
return
postJsonObject['cc']=postJsonObject['actor']+'/followers'
elif isinstance(postJsonObject['object'], dict):
if not postJsonObject['object'].get('to'):
return
if len(postJsonObject['object']['to'])>1:
return
if len(postJsonObject['object']['to'])==0:
return
if not postJsonObject['object']['to'][0].endswith('#Public'):
return
if postJsonObject['object'].get('cc'):
return
postJsonObject['object']['cc']=postJsonObject['actor']+'/followers'
def sendSignedJson(postJsonObject: {},session,baseDir: str, \
nickname: str, domain: str, port: int, \
toNickname: str, toDomain: str, toPort: int, cc: str, \
httpPrefix: str, saveToFile: bool, clientToServer: bool, \
federationList: [], \
sendThreads: [], postLog: [], cachedWebfingers: {}, \
personCache: {}, debug: bool,projectVersion: str) -> int:
"""Sends a signed json object to an inbox/outbox
"""
if debug:
print('DEBUG: sendSignedJson start')
if not session:
print('WARN: No session specified for sendSignedJson')
return 8
withDigest=True
sharedInbox=False
if toNickname=='inbox':
# shared inbox actor on @domain@domain
toNickname=toDomain
sharedInbox=True
toDomainOriginal=toDomain
if toPort:
if toPort!=80 and toPort!=443:
if ':' not in toDomain:
toDomain=toDomain+':'+str(toPort)
handle=httpPrefix+'://'+toDomain+'/@'+toNickname
if debug:
print('DEBUG: handle - '+handle+' toPort '+str(toPort))
# lookup the inbox for the To handle
wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
domain,projectVersion)
if not wfRequest:
if debug:
print('DEBUG: webfinger for '+handle+' failed')
return 1
if wfRequest.get('errors'):
if debug:
print('DEBUG: webfinger for '+handle+' failed with errors '+str(wfRequest['errors']))
if not clientToServer:
postToBox='inbox'
else:
postToBox='outbox'
# get the actor inbox/outbox/capabilities for the To handle
inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
getPersonBox(baseDir,session,wfRequest,personCache, \
projectVersion,httpPrefix,nickname,domain,postToBox)
if nickname=='capabilities':
inboxUrl=capabilityAcquisition
if not capabilityAcquisition:
return 2
else:
print("inboxUrl: "+str(inboxUrl))
print("toPersonId: "+str(toPersonId))
print("sharedInboxUrl: "+str(sharedInboxUrl))
if inboxUrl:
if inboxUrl.endswith('/actor/inbox'):
inboxUrl=sharedInboxUrl
if not inboxUrl:
if debug:
print('DEBUG: missing inboxUrl')
return 3
if debug:
print('DEBUG: Sending to endpoint '+inboxUrl)
if not pubKey:
if debug:
print('DEBUG: missing pubkey')
return 4
if not toPersonId:
if debug:
print('DEBUG: missing personId')
return 5
# sharedInbox and capabilities are optional
# get the senders private key
privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
if len(privateKeyPem)==0:
if debug:
print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
return 6
if toDomain not in inboxUrl:
if debug:
print('DEBUG: '+toDomain+' is not in '+inboxUrl)
return 7
postPath=inboxUrl.split(toDomain,1)[1]
addFollowersToPublicPost(postJsonObject)
# convert json to string so that there are no
# subsequent conversions after creating message body digest
postJsonStr=json.dumps(postJsonObject)
# construct the http header, including the message body digest
signatureHeaderJson = \
createSignedHeader(privateKeyPem,nickname,domain,port, \
toDomain,toPort, \
postPath,httpPrefix,withDigest,postJsonStr)
# Keep the number of threads being used small
while len(sendThreads)>1000:
print('WARN: Maximum threads reached - killing send thread')
sendThreads[0].kill()
sendThreads.pop(0)
print('WARN: thread killed')
if debug:
print('DEBUG: starting thread to send post')
pprint(postJsonObject)
thr = threadWithTrace(target=threadSendPost, \
args=(session, \
postJsonStr, \
federationList, \
inboxUrl,baseDir, \
signatureHeaderJson.copy(), \
postLog,
debug),daemon=True)
sendThreads.append(thr)
#thr.start()
return 0
def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
"""The Follow activity doesn't have a 'to' field and so one
needs to be added so that activity distribution happens in a consistent way
Returns true if a 'to' field exists or was added
"""
if postJsonObject.get('to'):
return postJsonObject,True
if debug:
pprint(postJsonObject)
print('DEBUG: no "to" field when sending to named addresses 2')
isSameType=False
toFieldAdded=False
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], str):
if postJsonObject.get('type'):
if postJsonObject['type']==activityType:
isSameType=True
if debug:
print('DEBUG: "to" field assigned to Follow')
toAddress=postJsonObject['object']
if '/statuses/' in toAddress:
toAddress=toAddress.split('/statuses/')[0]
postJsonObject['to']=[toAddress]
toFieldAdded=True
elif isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('type'):
if postJsonObject['object']['type']==activityType:
isSameType=True
if isinstance(postJsonObject['object']['object'], str):
if debug:
print('DEBUG: "to" field assigned to Follow')
toAddress=postJsonObject['object']['object']
if '/statuses/' in toAddress:
toAddress=toAddress.split('/statuses/')[0]
postJsonObject['object']['to']=[toAddress]
postJsonObject['to']=[postJsonObject['object']['object']]
toFieldAdded=True
if not isSameType:
return postJsonObject,True
if toFieldAdded:
return postJsonObject,True
return postJsonObject,False
def sendToNamedAddresses(session,baseDir: str, \
nickname: str, domain: str, port: int, \
httpPrefix: str,federationList: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
postJsonObject: {},debug: bool, \
projectVersion: str) -> None:
"""sends a post to the specific named addresses in to/cc
"""
if not session:
print('WARN: No session for sendToNamedAddresses')
return
if not postJsonObject.get('object'):
return
if isinstance(postJsonObject['object'], dict):
isProfileUpdate=False
# for actor updates there is no 'to' within the object
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
if postJsonObject['type']=='Update' and \
(postJsonObject['object']['type']=='Person' or \
postJsonObject['object']['type']=='Application' or \
postJsonObject['object']['type']=='Group' or \
postJsonObject['object']['type']=='Service'):
# use the original object, which has a 'to'
recipientsObject=postJsonObject
isProfileUpdate=True
if not isProfileUpdate:
if not postJsonObject['object'].get('to'):
if debug:
pprint(postJsonObject)
print('DEBUG: no "to" field when sending to named addresses')
if postJsonObject['object'].get('type'):
if postJsonObject['object']['type']=='Follow':
if isinstance(postJsonObject['object']['object'], str):
if debug:
print('DEBUG: "to" field assigned to Follow')
postJsonObject['object']['to']=[postJsonObject['object']['object']]
if not postJsonObject['object'].get('to'):
return
recipientsObject=postJsonObject['object']
else:
postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
if not fieldAdded:
return
postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
if not fieldAdded:
return
recipientsObject=postJsonObject
recipients=[]
recipientType=['to','cc']
for rType in recipientType:
if not recipientsObject.get(rType):
continue
if isinstance(recipientsObject[rType], list):
if debug:
pprint(recipientsObject)
print('recipientsObject: '+str(recipientsObject[rType]))
for address in recipientsObject[rType]:
if not address:
continue
if '/' not in address:
continue
if address.endswith('#Public'):
continue
if address.endswith('/followers'):
continue
recipients.append(address)
elif isinstance(recipientsObject[rType], str):
address=recipientsObject[rType]
if address:
if '/' in address:
if address.endswith('#Public'):
continue
if address.endswith('/followers'):
continue
recipients.append(address)
if not recipients:
if debug:
print('DEBUG: no individual recipients')
return
if debug:
print('DEBUG: Sending individually addressed posts: '+str(recipients))
# this is after the message has arrived at the server
clientToServer=False
for address in recipients:
toNickname=getNicknameFromActor(address)
if not toNickname:
continue
toDomain,toPort=getDomainFromActor(address)
if not toDomain:
continue
if debug:
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
toDomainFull=toDomain
if toPort:
if toPort!=80 and toPort!=443:
if ':' not in toDomain:
toDomainFull=toDomain+':'+str(toPort)
print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
cc=[]
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
toNickname,toDomain,toPort, \
cc,httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug,projectVersion)
def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
"""Returns true if the given domain has a shared inbox
"""
wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
None,__version__)
if wfRequest:
if not wfRequest.get('errors'):
return True
return False
def sendToFollowers(session,baseDir: str, \
nickname: str, domain: str, port: int, \
httpPrefix: str,federationList: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
postJsonObject: {},debug: bool, \
projectVersion: str) -> None:
"""sends a post to the followers of the given nickname
"""
print('sendToFollowers')
if not session:
print('WARN: No session for sendToFollowers')
return
if not postIsAddressedToFollowers(baseDir,nickname,domain, \
port,httpPrefix,postJsonObject):
if debug:
print('Post is not addressed to followers')
return
print('Post is addressed to followers')
grouped=groupFollowersByDomain(baseDir,nickname,domain)
if not grouped:
if debug:
print('Post to followers did not resolve any domains')
return
print('Post to followers resolved domains')
print(str(grouped))
# this is after the message has arrived at the server
clientToServer=False
# for each instance
for followerDomain,followerHandles in grouped.items():
if debug:
print('DEBUG: follower handles for '+followerDomain)
pprint(followerHandles)
withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
if debug:
if withSharedInbox:
print(followerDomain+' has shared inbox')
else:
print(followerDomain+' does not have a shared inbox')
toPort=port
index=0
toDomain=followerHandles[index].split('@')[1]
if ':' in toDomain:
toPort=toDomain.split(':')[1]
toDomain=toDomain.split(':')[0]
# If this is a profile update then send to shared inbox
cc=''
if withSharedInbox:
toNickname=followerHandles[index].split('@')[0]
# if there are more than one followers on the domain
# then send the post to the shared inbox
if len(followerHandles)>1:
toNickname='inbox'
if toNickname!='inbox' and postJsonObject.get('type'):
if postJsonObject['type']=='Update':
if postJsonObject.get('object'):
if isinstance(postJsonObject['object'], dict):
if postJsonObject['object'].get('type'):
if postJsonObject['object']['type']=='Person' or \
postJsonObject['object']['type']=='Application' or \
postJsonObject['object']['type']=='Group' or \
postJsonObject['object']['type']=='Service':
print('Sending profile update to shared inbox of '+toDomain)
toNickname='inbox'
if debug:
print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
toNickname,toDomain,toPort, \
cc,httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug,projectVersion)
else:
# send to individual followers without using a shared inbox
for handle in followerHandles:
toNickname=handle.split('@')[0]
if debug:
print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
sendSignedJson(postJsonObject,session,baseDir, \
nickname,domain,port, \
toNickname,toDomain,toPort, \
cc,httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug,projectVersion)
if debug:
print('DEBUG: End of sendToFollowers')
def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxBase(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxBase(session,baseDir,'dm',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxBase(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
return createBoxBase(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
return createBoxBase(session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
itemsPerPage,headerOnly,authorized,False,pageNumber)
def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
boxname='moderation'
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
if not pageNumber:
pageNumber=1
pageStr='?page='+str(pageNumber)
boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'totalItems': 0,
'type': 'OrderedCollection'}
boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
'orderedItems': [
],
'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'type': 'OrderedCollectionPage'}
if isModerator(baseDir,nickname):
moderationIndexFile=baseDir+'/accounts/moderation.txt'
if os.path.isfile(moderationIndexFile):
with open(moderationIndexFile, "r") as f:
lines = f.readlines()
boxHeader['totalItems']=len(lines)
if headerOnly:
return boxHeader
pageLines=[]
if len(lines)>0:
endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
if endLineNumber<0:
endLineNumber=0
startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
if startLineNumber<0:
startLineNumber=0
lineNumber=startLineNumber
while lineNumber>=endLineNumber:
pageLines.append(lines[lineNumber].strip('\n'))
lineNumber-=1
for postUrl in pageLines:
postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
if os.path.isfile(postFilename):
tries=0
while tries<5:
try:
with open(postFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
boxItems['orderedItems'].append(postJsonObject)
break
except Exception as e:
print('WARN: commentjson exception createModeration - '+str(e))
time.sleep(1)
tries+=1
if headerOnly:
return boxHeader
return boxItems
def getStatusNumberFromPostFilename(filename) -> int:
"""Gets the status number from a post filename
eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
returns 156295850695206
"""
if '#statuses#' not in filename:
return None
return int(filename.split('#')[-1].replace('.json',''))
def isDM(postJsonObject: {}) -> bool:
"""Returns true if the given post is a DM
"""
if postJsonObject['type']!='Create':
return False
if not postJsonObject.get('object'):
return False
if not isinstance(postJsonObject['object'], dict):
return False
if postJsonObject['object']['type']!='Note':
return False
fields=['to','cc']
for f in fields:
if not postJsonObject['object'].get(f):
continue
for toAddress in postJsonObject['object'][f]:
if toAddress.endswith('#Public'):
return False
if toAddress.endswith('followers'):
return False
return True
def isImageMedia(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {}) -> bool:
"""Returns true if the given post has attached image media
"""
if postJsonObject['type']=='Announce':
postJsonAnnounce=downloadAnnounce(session,baseDir,httpPrefix,nickname,domain,postJsonObject,__version__)
if postJsonAnnounce:
postJsonObject=postJsonAnnounce
if postJsonObject['type']!='Create':
return False
if not postJsonObject.get('object'):
return False
if not isinstance(postJsonObject['object'], dict):
return False
if postJsonObject['object']['type']!='Note':
return False
if not postJsonObject['object'].get('attachment'):
return False
if not isinstance(postJsonObject['object']['attachment'], list):
return False
for attach in postJsonObject['object']['attachment']:
if attach.get('mediaType') and attach.get('url'):
if attach['mediaType'].startswith('image/'):
return True
return False
def isReply(postJsonObject: {},actor: str) -> bool:
"""Returns true if the given post is a reply to the given actor
"""
if postJsonObject['type']!='Create':
return False
if not postJsonObject.get('object'):
return False
if not isinstance(postJsonObject['object'], dict):
return False
if postJsonObject['object']['type']!='Note':
return False
if not postJsonObject['object'].get('inReplyTo'):
return False
if not postJsonObject['object']['inReplyTo'].startswith(actor):
return False
return True
def createBoxBase(session,baseDir: str,boxname: str, \
nickname: str,domain: str,port: int,httpPrefix: str, \
itemsPerPage: int,headerOnly: bool,authorized :bool, \
ocapAlways: bool,pageNumber=None) -> {}:
"""Constructs the box feed for a person with the given nickname
"""
if boxname!='inbox' and boxname!='dm' and \
boxname!='tlreplies' and boxname!='tlmedia' and boxname!='outbox':
return None
if boxname!='dm' and boxname!='tlreplies' and boxname!='tlmedia':
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
else:
# extract DMs or replies or media from the inbox
boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
sharedBoxDir=None
if boxname=='inbox' or boxname=='tlreplies' or boxname=='tlmedia':
sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
boxActor=httpPrefix+'://'+domain+'/users/'+nickname
pageStr='?page=true'
if pageNumber:
try:
pageStr='?page='+str(pageNumber)
except:
pass
boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
'totalItems': 0,
'type': 'OrderedCollection'}
boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
'orderedItems': [
],
'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
'type': 'OrderedCollectionPage'}
# counter for posts so far added to the target page
postsOnPageCtr=0
# post filenames sorted in descending order
postsInBoxDict={}
postsCtr=0
postsInPersonInbox=os.scandir(boxDir)
for postFilename in postsInPersonInbox:
postFilename=postFilename.name
if not postFilename.endswith('.json'):
continue
# extract the status number
statusNumber=getStatusNumberFromPostFilename(postFilename)
if statusNumber:
postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
postsCtr+=1
# combine the inbox for the account with the shared inbox
if sharedBoxDir:
handle=nickname+'@'+domain
followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
postsInSharedInbox=os.scandir(sharedBoxDir)
for postFilename in postsInSharedInbox:
postFilename=postFilename.name
statusNumber=getStatusNumberFromPostFilename(postFilename)
if statusNumber:
sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
# get the actor from the shared post
loadedPost=False
tries=0
while tries<5:
try:
with open(sharedInboxFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
loadedPost=True
break
except Exception as e:
print('WARN: commentjson exception createBoxBase - '+str(e))
time.sleep(1)
tries+=1
if loadedPost:
actorNickname=getNicknameFromActor(postJsonObject['actor'])
actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
if actorNickname and actorDomain:
# is the actor followed by this account?
if actorNickname+'@'+actorDomain in open(followingFilename).read():
if ocapAlways:
capsList=None
# Note: should this be in the Create or the object of a post?
if postJsonObject.get('capability'):
if isinstance(postJsonObject['capability'], list):
capsList=postJsonObject['capability']
# Have capabilities been granted for the sender?
ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
if os.path.isfile(ocapFilename):
# read the capabilities id
loadedOcap=False
tries=0
while tries<5:
try:
with open(ocapFilename, 'r') as fp:
ocapJson=commentjson.load(fp)
loadedOcap=True
break
except Exception as e:
print('WARN: commentjson exception createBoxBase - '+str(e))
time.sleep(1)
tries+=1
if loadedOcap:
if ocapJson.get('id'):
if ocapJson['id'] in capsList:
postsInBoxDict[statusNumber]=sharedInboxFilename
postsCtr+=1
else:
postsInBoxDict[statusNumber]=sharedInboxFilename
postsCtr+=1
# sort the list in descending order of date
postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=True))
# number of posts in box
boxHeader['totalItems']=postsCtr
prevPostFilename=None
if not pageNumber:
pageNumber=1
# Generate first and last entries within header
if postsCtr>0:
lastPage=int(postsCtr/itemsPerPage)
if lastPage<1:
lastPage=1
boxHeader['last']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
# Insert posts
currPage=1
postsCtr=0
for statusNumber,postFilename in postsInBox.items():
# Are we at the starting page yet?
if prevPostFilename and currPage==pageNumber and postsCtr==0:
# update the prev entry for the last message id
postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
boxHeader['prev']= \
httpPrefix+'://'+domain+'/users/'+nickname+'/'+ \
boxname+'?min_id='+postId+'&page=true'
# get the full path of the post file
filePath = postFilename
try:
if os.path.isfile(filePath):
# is this a valid timeline post?
isPost=False
# must be a "Note" or "Announce" type
with open(filePath, 'r') as file:
postStr = file.read()
if '"Note"' in postStr or '"Announce"' in postStr or \
('"Question"' in postStr and '"Create"' in postStr):
isPost=True
if boxname=='dm':
if '#Public' in postStr or '/followers' in postStr:
isPost=False
elif boxname=='tlreplies':
if boxActor not in postStr:
isPost=False
elif boxname=='tlmedia':
if '"Create"' in postStr:
if 'mediaType' not in postStr or 'image/' not in postStr:
isPost=False
if isPost:
isTimelinePost=False
if currPage <= pageNumber and postsOnPageCtr <= itemsPerPage:
# get the post as json
p = json.loads(postStr)
if (boxname!='dm' and boxname!='tlreplies' and boxname!='tlmedia'):
isTimelinePost=True
else:
if boxname=='dm':
if isDM(p):
isTimelinePost=True
elif boxname=='tlreplies':
if isDM(p) or isReply(p,boxActor):
isTimelinePost=True
elif boxname=='tlmedia':
if isImageMedia(session,baseDir,httpPrefix,nickname,domain,p):
isTimelinePost=True
if isTimelinePost and currPage == pageNumber:
# remove any capability so that it's not displayed
if p.get('capability'):
del p['capability']
# Don't show likes or replies to unauthorized viewers
if not authorized:
if p.get('object'):
if isinstance(p['object'], dict):
if p['object'].get('likes'):
p['likes']={}
if p['object'].get('replies'):
p['replies']={}
# insert it into the box feed
if postsOnPageCtr < itemsPerPage:
if not headerOnly:
boxItems['orderedItems'].append(p)
postsOnPageCtr += 1
if postsOnPageCtr >= itemsPerPage:
# if this is the last post update the next message ID
if '/statuses/' in p['id']:
postId = p['id'].split('/statuses/')[1].replace('/activity','')
boxHeader['next']= \
httpPrefix+'://'+domain+'/users/'+ \
nickname+'/'+boxname+'?max_id='+ \
postId+'&page=true'
# remember the last post filename for use with prev
prevPostFilename = postFilename
if postsOnPageCtr >= itemsPerPage:
break
# count the pages
if isTimelinePost:
postsCtr += 1
if postsCtr >= itemsPerPage:
postsCtr = 0
currPage += 1
except Exception as e:
print(e)
if headerOnly:
return boxHeader
return boxItems
def expireCache(baseDir: str,personCache: {},httpPrefix: str,archiveDir: str,maxPostsInBox=32000):
"""Thread used to expire actors from the cache and archive old posts
"""
while True:
# once per day
time.sleep(60*60*24)
expirePersonCache(basedir,personCache)
archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str,maxPostsInBox=32000) -> None:
"""Archives posts for all accounts
"""
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts'):
os.mkdir(archiveDir+'/accounts')
for subdir, dirs, files in os.walk(baseDir+'/accounts'):
for handle in dirs:
if '@' in handle:
nickname=handle.split('@')[0]
domain=handle.split('@')[1]
archiveSubdir=None
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts/'+handle):
os.mkdir(archiveDir+'/accounts/'+handle)
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
'inbox',archiveSubdir, \
maxPostsInBox)
if archiveDir:
archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
'outbox',archiveSubdir, \
maxPostsInBox)
def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
boxname: str,archiveDir: str,maxPostsInBox=32000) -> None:
"""Retain a maximum number of posts within the given box
Move any others to an archive directory
"""
if boxname!='inbox' and boxname!='outbox':
return
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
postsInBox=os.scandir(boxDir)
noOfPosts=0
for f in postsInBox:
noOfPosts+=1
if noOfPosts<=maxPostsInBox:
return
postsInBoxDict={}
postsCtr=0
for postFilename in postsInBox:
postFilename=postFilename.name
if not postFilename.endswith('.json'):
continue
# extract the status number
statusNumber=getStatusNumberFromPostFilename(postFilename)
if statusNumber:
postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
postsCtr+=1
noOfPosts=postsCtr
if noOfPosts<=maxPostsInBox:
return
# sort the list in ascending order of date
postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=False))
# directory containing cached html posts
postCacheDir=boxDir.replace('/'+boxname,'/postcache')
for statusNumber,postFilename in postsInBox.items():
filePath = os.path.join(boxDir, postFilename)
if not os.path.isfile(filePath):
continue
if archiveDir:
repliesPath=filePath.replace('.json','.replies')
archivePath = os.path.join(archiveDir, postFilename)
os.rename(filePath,archivePath)
if os.path.isfile(repliesPath):
os.rename(repliesPath,archivePath)
else:
deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
# remove cached html posts
postCacheFilename=os.path.join(postCacheDir, postFilename).replace('.json','.html')
if os.path.isfile(postCacheFilename):
os.remove(postCacheFilename)
noOfPosts -= 1
if noOfPosts <= maxPostsInBox:
break
def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
raw: bool,simple: bool,useTor: bool, \
port: int,httpPrefix: str, \
debug: bool,projectVersion: str) -> None:
""" This is really just for test purposes
"""
session = createSession(domain,port,useTor)
personCache={}
cachedWebfingers={}
federationList=[]
domainFull=domain
if port:
if port!=80 and port!=443:
if ':' not in domain:
domainFull=domain+':'+str(port)
handle=httpPrefix+"://"+domainFull+"/@"+nickname
wfRequest = \
webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
domain,projectVersion)
if not wfRequest:
sys.exit()
personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
getPersonBox(baseDir,session,wfRequest,personCache, \
projectVersion,httpPrefix,nickname,domain,'outbox')
wfResult = json.dumps(wfRequest, indent=2, sort_keys=False)
maxMentions=10
maxEmoji=10
maxAttachments=5
userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
maxAttachments,federationList, \
personCache,raw,simple,debug, \
projectVersion,httpPrefix,domain)
#print(str(userPosts))
def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
nickname: str,domain: str,port: int, \
followerUrl,updateCaps: [], \
sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {}, \
federationList :[],debug :bool, \
projectVersion: str) -> int:
"""When the capabilities for a follower are changed this
sends out an update. followerUrl is the actor of the follower.
"""
updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
nickname,domain,port, \
followerUrl, \
updateCaps)
if not updateJson:
return 1
if debug:
pprint(updateJson)
print('DEBUG: sending capabilities update from '+ \
nickname+'@'+domain+' port '+ str(port) + \
' to '+followerUrl)
clientToServer=False
followerNickname=getNicknameFromActor(followerUrl)
if not followerNickname:
print('WARN: unable to find nickname in '+followerUrl)
return 1
followerDomain,followerPort=getDomainFromActor(followerUrl)
return sendSignedJson(updateJson,session,baseDir, \
nickname,domain,port, \
followerNickname,followerDomain,followerPort, '', \
httpPrefix,True,clientToServer, \
federationList, \
sendThreads,postLog,cachedWebfingers, \
personCache,debug,projectVersion)
def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
postRepliesFilename: str,authorized: bool, \
repliesJson: {}) -> None:
# populate the items list with replies
repliesBoxes=['outbox','inbox']
with open(postRepliesFilename,'r') as repliesFile:
for messageId in repliesFile:
replyFound=False
# examine inbox and outbox
for boxname in repliesBoxes:
searchFilename= \
baseDir+ \
'/accounts/'+nickname+'@'+ \
domain+'/'+ \
boxname+'/'+ \
messageId.replace('\n','').replace('/','#')+'.json'
if os.path.isfile(searchFilename):
if authorized or \
'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
loadedPost=False
tries=0
while tries<5:
try:
with open(searchFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
loadedPost=True
break
except Exception as e:
print('WARN: commentjson exception populateRepliesJson - '+str(e))
time.sleep(1)
tries+=1
if loadedPost:
if postJsonObject['object'].get('cc'):
if authorized or \
('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
repliesJson['orderedItems'].append(postJsonObject)
replyFound=True
else:
if authorized or \
'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
repliesJson['orderedItems'].append(postJsonObject)
replyFound=True
break
# if not in either inbox or outbox then examine the shared inbox
if not replyFound:
searchFilename= \
baseDir+ \
'/accounts/inbox@'+ \
domain+'/inbox/'+ \
messageId.replace('\n','').replace('/','#')+'.json'
if os.path.isfile(searchFilename):
if authorized or \
'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
# get the json of the reply and append it to the collection
loadedPost=False
tries=0
while tries<5:
try:
with open(searchFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
loadedPost=True
break
except Exception as e:
print('WARN: commentjson exception populateRepliesJson 2 - '+str(e))
time.sleep(1)
tries+=1
if loadedPost:
if postJsonObject['object'].get('cc'):
if authorized or \
('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
repliesJson['orderedItems'].append(postJsonObject)
else:
if authorized or \
'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
repliesJson['orderedItems'].append(postJsonObject)
def rejectAnnounce(announceFilename: str):
"""Marks an announce as rejected
"""
if not os.path.isfile(announceFilename+'.reject'):
rejectAnnounceFile=open(announceFilename+'.reject', "w+")
rejectAnnounceFile.write('\n')
rejectAnnounceFile.close()
def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {},projectVersion: str) -> {}:
"""Download the post referenced by an announce
"""
if not postJsonObject.get('object'):
return None
if not isinstance(postJsonObject['object'], str):
return None
# get the announced post
announceCacheDir=baseDir+'/cache/announce/'+nickname
if not os.path.isdir(announceCacheDir):
os.mkdir(announceCacheDir)
announceFilename=announceCacheDir+'/'+postJsonObject['object'].replace('/','#')+'.json'
print('announceFilename: '+announceFilename)
if os.path.isfile(announceFilename+'.reject'):
return None
if os.path.isfile(announceFilename):
print('Reading cached Announce content for '+postJsonObject['object'])
tries=0
while tries<5:
try:
with open(announceFilename, 'r') as fp:
postJsonObject=commentjson.load(fp)
return postJsonObject
except Exception as e:
print('WARN: commentjson exception downloadAnnounce - '+str(e))
time.sleep(1)
tries+=1
else:
print('Downloading Announce content for '+postJsonObject['object'])
asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
if '/channel/' in postJsonObject['actor']:
asHeader={'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
actorNickname=getNicknameFromActor(postJsonObject['actor'])
actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
announcedJson=getJson(session,postJsonObject['object'],asHeader,None,projectVersion,httpPrefix,domain)
if not announcedJson:
return None
if not announcedJson.get('id'):
rejectAnnounce(announceFilename)
pprint(announcedJson)
return None
if '/statuses/' not in announcedJson['id']:
rejectAnnounce(announceFilename)
return None
if '/users/' not in announcedJson['id'] and \
'/channel/' not in announcedJson['id'] and \
'/profile/' not in announcedJson['id']:
rejectAnnounce(announceFilename)
return None
if not announcedJson.get('type'):
rejectAnnounce(announceFilename)
pprint(announcedJson)
return None
if announcedJson['type']!='Note':
rejectAnnounce(announceFilename)
pprint(announcedJson)
return None
# wrap in create to be consistent with other posts
announcedJson= \
outboxMessageCreateWrap(httpPrefix, \
actorNickname,actorDomain,actorPort, \
announcedJson)
if announcedJson['type']!='Create':
rejectAnnounce(announceFilename)
pprint(announcedJson)
return None
# set the id to the original status
announcedJson['id']=postJsonObject['object']
announcedJson['object']['id']=postJsonObject['object']
# check that the repeat isn't for a blocked account
attributedNickname=getNicknameFromActor(announcedJson['object']['id'])
attributedDomain,attributedPort=getDomainFromActor(announcedJson['object']['id'])
if attributedNickname and attributedDomain:
if attributedPort:
if attributedPort!=80 and attributedPort!=443:
attributedDomain=attributedDomain+':'+str(attributedPort)
if isBlocked(baseDir,nickname,domain,attributedNickname,attributedDomain):
rejectAnnounce(announceFilename)
return None
postJsonObject=announcedJson
tries=0
while tries<5:
try:
with open(announceFilename, 'w') as fp:
commentjson.dump(postJsonObject, fp, indent=2, sort_keys=False)
return postJsonObject
except Exception as e:
print(e)
time.sleep(1)
tries+=1
return None
Reference in New Issue View Git Blame Copy Permalink