forked from indymedia/epicyon
4038 lines
153 KiB
Python
4038 lines
153 KiB
Python
__filename__ = "posts.py"
|
||
__author__ = "Bob Mottram"
|
||
__license__ = "AGPL3+"
|
||
__version__ = "1.1.0"
|
||
__maintainer__ = "Bob Mottram"
|
||
__email__ = "bob@freedombone.net"
|
||
__status__ = "Production"
|
||
|
||
import json
|
||
import html
|
||
import datetime
|
||
import os
|
||
import shutil
|
||
import sys
|
||
import time
|
||
import uuid
|
||
import random
|
||
from socket import error as SocketError
|
||
from time import gmtime, strftime
|
||
from collections import OrderedDict
|
||
from threads import threadWithTrace
|
||
from cache import storePersonInCache
|
||
from cache import getPersonFromCache
|
||
from cache import expirePersonCache
|
||
from pprint import pprint
|
||
from session import createSession
|
||
from session import getJson
|
||
from session import postJson
|
||
from session import postJsonString
|
||
from session import postImage
|
||
from webfinger import webfingerHandle
|
||
from httpsig import createSignedHeader
|
||
from utils import getFollowersList
|
||
from utils import isEvil
|
||
from utils import removeIdEnding
|
||
from utils import siteIsActive
|
||
from utils import getCachedPostFilename
|
||
from utils import getStatusNumber
|
||
from utils import createPersonDir
|
||
from utils import urlPermitted
|
||
from utils import getNicknameFromActor
|
||
from utils import getDomainFromActor
|
||
from utils import deletePost
|
||
from utils import validNickname
|
||
from utils import locatePost
|
||
from utils import loadJson
|
||
from utils import saveJson
|
||
from utils import getConfigParam
|
||
from utils import locateNewsVotes
|
||
from utils import locateNewsArrival
|
||
from utils import votesOnNewswireItem
|
||
from utils import removeHtml
|
||
from media import attachMedia
|
||
from media import replaceYouTube
|
||
from content import removeLongWords
|
||
from content import addHtmlTags
|
||
from content import replaceEmojiFromTags
|
||
from content import removeTextFormatting
|
||
from auth import createBasicAuthHeader
|
||
from blocking import isBlocked
|
||
from filters import isFiltered
|
||
from git import convertPostToPatch
|
||
from jsonldsig import jsonldSign
|
||
from petnames import resolvePetnames
|
||
|
||
|
||
def isModerator(baseDir: str, nickname: str) -> bool:
|
||
"""Returns true if the given nickname is a moderator
|
||
"""
|
||
moderatorsFile = baseDir + '/accounts/moderators.txt'
|
||
|
||
if not os.path.isfile(moderatorsFile):
|
||
adminName = getConfigParam(baseDir, 'admin')
|
||
if not adminName:
|
||
return False
|
||
if adminName == nickname:
|
||
return True
|
||
return False
|
||
|
||
with open(moderatorsFile, "r") as f:
|
||
lines = f.readlines()
|
||
if len(lines) == 0:
|
||
adminName = getConfigParam(baseDir, 'admin')
|
||
if not adminName:
|
||
return False
|
||
if adminName == nickname:
|
||
return True
|
||
for moderator in lines:
|
||
moderator = moderator.strip('\n').strip('\r')
|
||
if moderator == nickname:
|
||
return True
|
||
return False
|
||
|
||
|
||
def noOfFollowersOnDomain(baseDir: str, handle: str,
|
||
domain: str, followFile='followers.txt') -> int:
|
||
"""Returns the number of followers of the given handle from the given domain
|
||
"""
|
||
filename = baseDir + '/accounts/' + handle + '/' + followFile
|
||
if not os.path.isfile(filename):
|
||
return 0
|
||
|
||
ctr = 0
|
||
with open(filename, "r") as followersFilename:
|
||
for followerHandle in followersFilename:
|
||
if '@' in followerHandle:
|
||
followerDomain = followerHandle.split('@')[1]
|
||
followerDomain = followerDomain.replace('\n', '')
|
||
followerDomain = followerDomain.replace('\r', '')
|
||
if domain == followerDomain:
|
||
ctr += 1
|
||
return ctr
|
||
|
||
|
||
def getPersonKey(nickname: str, domain: str, baseDir: str, keyType='public',
|
||
debug=False):
|
||
"""Returns the public or private key of a person
|
||
"""
|
||
handle = nickname + '@' + domain
|
||
keyFilename = baseDir + '/keys/' + keyType + '/' + handle.lower() + '.key'
|
||
if not os.path.isfile(keyFilename):
|
||
if debug:
|
||
print('DEBUG: private key file not found: ' + keyFilename)
|
||
return ''
|
||
keyPem = ''
|
||
with open(keyFilename, "r") as pemFile:
|
||
keyPem = pemFile.read()
|
||
if len(keyPem) < 20:
|
||
if debug:
|
||
print('DEBUG: private key was too short: ' + keyPem)
|
||
return ''
|
||
return keyPem
|
||
|
||
|
||
def cleanHtml(rawHtml: str) -> str:
|
||
# text=BeautifulSoup(rawHtml, 'html.parser').get_text()
|
||
text = rawHtml
|
||
return html.unescape(text)
|
||
|
||
|
||
def getUserUrl(wfRequest: {}) -> str:
|
||
if wfRequest.get('links'):
|
||
for link in wfRequest['links']:
|
||
if link.get('type') and link.get('href'):
|
||
if link['type'] == 'application/activity+json':
|
||
if not ('/users/' in link['href'] or
|
||
'/accounts/' in link['href'] or
|
||
'/profile/' in link['href'] or
|
||
'/channel/' in link['href']):
|
||
print('Webfinger activity+json contains ' +
|
||
'single user instance actor')
|
||
return link['href']
|
||
return None
|
||
|
||
|
||
def parseUserFeed(session, feedUrl: str, asHeader: {},
|
||
projectVersion: str, httpPrefix: str,
|
||
domain: str, depth=0) -> {}:
|
||
if depth > 10:
|
||
return None
|
||
|
||
feedJson = getJson(session, feedUrl, asHeader, None,
|
||
projectVersion, httpPrefix, domain)
|
||
if not feedJson:
|
||
return None
|
||
|
||
if 'orderedItems' in feedJson:
|
||
for item in feedJson['orderedItems']:
|
||
yield item
|
||
|
||
nextUrl = None
|
||
if 'first' in feedJson:
|
||
nextUrl = feedJson['first']
|
||
elif 'next' in feedJson:
|
||
nextUrl = feedJson['next']
|
||
|
||
if nextUrl:
|
||
if isinstance(nextUrl, str):
|
||
if '?max_id=0' not in nextUrl:
|
||
userFeed = \
|
||
parseUserFeed(session, nextUrl, asHeader,
|
||
projectVersion, httpPrefix,
|
||
domain, depth+1)
|
||
if userFeed:
|
||
for item in userFeed:
|
||
yield item
|
||
elif isinstance(nextUrl, dict):
|
||
userFeed = nextUrl
|
||
if userFeed.get('orderedItems'):
|
||
for item in userFeed['orderedItems']:
|
||
yield item
|
||
|
||
|
||
def getPersonBox(baseDir: str, session, wfRequest: {},
|
||
personCache: {},
|
||
projectVersion: str, httpPrefix: str,
|
||
nickname: str, domain: str,
|
||
boxName='inbox') -> (str, str, str, str, str, str, str, str):
|
||
profileStr = 'https://www.w3.org/ns/activitystreams'
|
||
asHeader = {
|
||
'Accept': 'application/activity+json; profile="' + profileStr + '"'
|
||
}
|
||
if not wfRequest.get('errors'):
|
||
personUrl = getUserUrl(wfRequest)
|
||
else:
|
||
if nickname == 'dev':
|
||
# try single user instance
|
||
print('getPersonBox: Trying single user instance with ld+json')
|
||
personUrl = httpPrefix + '://' + domain
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
else:
|
||
personUrl = httpPrefix + '://' + domain + '/users/' + nickname
|
||
if not personUrl:
|
||
return None, None, None, None, None, None, None
|
||
personJson = \
|
||
getPersonFromCache(baseDir, personUrl, personCache, True)
|
||
if not personJson:
|
||
if '/channel/' in personUrl or '/accounts/' in personUrl:
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
personJson = getJson(session, personUrl, asHeader, None,
|
||
projectVersion, httpPrefix, domain)
|
||
if not personJson:
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
personJson = getJson(session, personUrl, asHeader, None,
|
||
projectVersion, httpPrefix, domain)
|
||
if not personJson:
|
||
print('Unable to get actor')
|
||
return None, None, None, None, None, None, None
|
||
boxJson = None
|
||
if not personJson.get(boxName):
|
||
if personJson.get('endpoints'):
|
||
if personJson['endpoints'].get(boxName):
|
||
boxJson = personJson['endpoints'][boxName]
|
||
else:
|
||
boxJson = personJson[boxName]
|
||
|
||
if not boxJson:
|
||
return None, None, None, None, None, None, None
|
||
|
||
personId = None
|
||
if personJson.get('id'):
|
||
personId = personJson['id']
|
||
pubKeyId = None
|
||
pubKey = None
|
||
if personJson.get('publicKey'):
|
||
if personJson['publicKey'].get('id'):
|
||
pubKeyId = personJson['publicKey']['id']
|
||
if personJson['publicKey'].get('publicKeyPem'):
|
||
pubKey = personJson['publicKey']['publicKeyPem']
|
||
sharedInbox = None
|
||
if personJson.get('sharedInbox'):
|
||
sharedInbox = personJson['sharedInbox']
|
||
else:
|
||
if personJson.get('endpoints'):
|
||
if personJson['endpoints'].get('sharedInbox'):
|
||
sharedInbox = personJson['endpoints']['sharedInbox']
|
||
avatarUrl = None
|
||
if personJson.get('icon'):
|
||
if personJson['icon'].get('url'):
|
||
avatarUrl = personJson['icon']['url']
|
||
displayName = None
|
||
if personJson.get('name'):
|
||
displayName = removeHtml(personJson['name'])
|
||
|
||
storePersonInCache(baseDir, personUrl, personJson, personCache, True)
|
||
|
||
return boxJson, pubKeyId, pubKey, personId, sharedInbox, \
|
||
avatarUrl, displayName
|
||
|
||
|
||
def getPosts(session, outboxUrl: str, maxPosts: int,
|
||
maxMentions: int,
|
||
maxEmoji: int, maxAttachments: int,
|
||
federationList: [],
|
||
personCache: {}, raw: bool,
|
||
simple: bool, debug: bool,
|
||
projectVersion: str, httpPrefix: str,
|
||
domain: str) -> {}:
|
||
"""Gets public posts from an outbox
|
||
"""
|
||
personPosts = {}
|
||
if not outboxUrl:
|
||
return personPosts
|
||
profileStr = 'https://www.w3.org/ns/activitystreams'
|
||
asHeader = {
|
||
'Accept': 'application/activity+json; profile="' + profileStr + '"'
|
||
}
|
||
if '/outbox/' in outboxUrl:
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
if raw:
|
||
result = []
|
||
i = 0
|
||
userFeed = parseUserFeed(session, outboxUrl, asHeader,
|
||
projectVersion, httpPrefix, domain)
|
||
for item in userFeed:
|
||
result.append(item)
|
||
i += 1
|
||
if i == maxPosts:
|
||
break
|
||
pprint(result)
|
||
return None
|
||
|
||
i = 0
|
||
userFeed = parseUserFeed(session, outboxUrl, asHeader,
|
||
projectVersion, httpPrefix, domain)
|
||
for item in userFeed:
|
||
if not item.get('id'):
|
||
if debug:
|
||
print('No id')
|
||
continue
|
||
if not item.get('type'):
|
||
if debug:
|
||
print('No type')
|
||
continue
|
||
if item['type'] != 'Create':
|
||
if debug:
|
||
print('Not Create type')
|
||
continue
|
||
if not item.get('object'):
|
||
if debug:
|
||
print('No object')
|
||
continue
|
||
if not isinstance(item['object'], dict):
|
||
if debug:
|
||
print('item object is not a dict')
|
||
continue
|
||
if not item['object'].get('published'):
|
||
if debug:
|
||
print('No published attribute')
|
||
continue
|
||
if not personPosts.get(item['id']):
|
||
# check that this is a public post
|
||
# #Public should appear in the "to" list
|
||
if item['object'].get('to'):
|
||
isPublic = False
|
||
for recipient in item['object']['to']:
|
||
if recipient.endswith('#Public'):
|
||
isPublic = True
|
||
break
|
||
if not isPublic:
|
||
continue
|
||
|
||
content = \
|
||
item['object']['content'].replace(''', "'")
|
||
|
||
mentions = []
|
||
emoji = {}
|
||
if item['object'].get('tag'):
|
||
for tagItem in item['object']['tag']:
|
||
tagType = tagItem['type'].lower()
|
||
if tagType == 'emoji':
|
||
if tagItem.get('name') and tagItem.get('icon'):
|
||
if tagItem['icon'].get('url'):
|
||
# No emoji from non-permitted domains
|
||
if urlPermitted(tagItem['icon']['url'],
|
||
federationList,
|
||
"objects:read"):
|
||
emojiName = tagItem['name']
|
||
emojiIcon = tagItem['icon']['url']
|
||
emoji[emojiName] = emojiIcon
|
||
else:
|
||
if debug:
|
||
print('url not permitted ' +
|
||
tagItem['icon']['url'])
|
||
if tagType == 'mention':
|
||
if tagItem.get('name'):
|
||
if tagItem['name'] not in mentions:
|
||
mentions.append(tagItem['name'])
|
||
if len(mentions) > maxMentions:
|
||
if debug:
|
||
print('max mentions reached')
|
||
continue
|
||
if len(emoji) > maxEmoji:
|
||
if debug:
|
||
print('max emojis reached')
|
||
continue
|
||
|
||
summary = ''
|
||
if item['object'].get('summary'):
|
||
if item['object']['summary']:
|
||
summary = item['object']['summary']
|
||
|
||
inReplyTo = ''
|
||
if item['object'].get('inReplyTo'):
|
||
if item['object']['inReplyTo']:
|
||
if isinstance(item['object']['inReplyTo'], str):
|
||
# No replies to non-permitted domains
|
||
if not urlPermitted(item['object']['inReplyTo'],
|
||
federationList,
|
||
"objects:read"):
|
||
if debug:
|
||
print('url not permitted ' +
|
||
item['object']['inReplyTo'])
|
||
continue
|
||
inReplyTo = item['object']['inReplyTo']
|
||
|
||
conversation = ''
|
||
if item['object'].get('conversation'):
|
||
if item['object']['conversation']:
|
||
# no conversations originated in non-permitted domains
|
||
if urlPermitted(item['object']['conversation'],
|
||
federationList, "objects:read"):
|
||
conversation = item['object']['conversation']
|
||
|
||
attachment = []
|
||
if item['object'].get('attachment'):
|
||
if item['object']['attachment']:
|
||
for attach in item['object']['attachment']:
|
||
if attach.get('name') and attach.get('url'):
|
||
# no attachments from non-permitted domains
|
||
if urlPermitted(attach['url'],
|
||
federationList,
|
||
"objects:read"):
|
||
attachment.append([attach['name'],
|
||
attach['url']])
|
||
else:
|
||
if debug:
|
||
print('url not permitted ' +
|
||
attach['url'])
|
||
|
||
sensitive = False
|
||
if item['object'].get('sensitive'):
|
||
sensitive = item['object']['sensitive']
|
||
|
||
if simple:
|
||
print(cleanHtml(content) + '\n')
|
||
else:
|
||
pprint(item)
|
||
personPosts[item['id']] = {
|
||
"sensitive": sensitive,
|
||
"inreplyto": inReplyTo,
|
||
"summary": summary,
|
||
"html": content,
|
||
"plaintext": cleanHtml(content),
|
||
"attachment": attachment,
|
||
"mentions": mentions,
|
||
"emoji": emoji,
|
||
"conversation": conversation
|
||
}
|
||
i += 1
|
||
|
||
if i == maxPosts:
|
||
break
|
||
return personPosts
|
||
|
||
|
||
def getPostDomains(session, outboxUrl: str, maxPosts: int,
|
||
maxMentions: int,
|
||
maxEmoji: int, maxAttachments: int,
|
||
federationList: [],
|
||
personCache: {},
|
||
debug: bool,
|
||
projectVersion: str, httpPrefix: str,
|
||
domain: str, domainList=[]) -> []:
|
||
"""Returns a list of domains referenced within public posts
|
||
"""
|
||
if not outboxUrl:
|
||
return []
|
||
profileStr = 'https://www.w3.org/ns/activitystreams'
|
||
asHeader = {
|
||
'Accept': 'application/activity+json; profile="' + profileStr + '"'
|
||
}
|
||
if '/outbox/' in outboxUrl:
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
|
||
postDomains = domainList
|
||
|
||
i = 0
|
||
userFeed = parseUserFeed(session, outboxUrl, asHeader,
|
||
projectVersion, httpPrefix, domain)
|
||
for item in userFeed:
|
||
i += 1
|
||
if i > maxPosts:
|
||
break
|
||
if not item.get('object'):
|
||
continue
|
||
if not isinstance(item['object'], dict):
|
||
continue
|
||
if item['object'].get('inReplyTo'):
|
||
if isinstance(item['object']['inReplyTo'], str):
|
||
postDomain, postPort = \
|
||
getDomainFromActor(item['object']['inReplyTo'])
|
||
if postDomain not in postDomains:
|
||
postDomains.append(postDomain)
|
||
|
||
if item['object'].get('tag'):
|
||
for tagItem in item['object']['tag']:
|
||
tagType = tagItem['type'].lower()
|
||
if tagType == 'mention':
|
||
if tagItem.get('href'):
|
||
postDomain, postPort = \
|
||
getDomainFromActor(tagItem['href'])
|
||
if postDomain not in postDomains:
|
||
postDomains.append(postDomain)
|
||
return postDomains
|
||
|
||
|
||
def deleteAllPosts(baseDir: str,
|
||
nickname: str, domain: str, boxname: str) -> None:
|
||
"""Deletes all posts for a person from inbox or outbox
|
||
"""
|
||
if boxname != 'inbox' and boxname != 'outbox' and \
|
||
boxname != 'tlblogs' and boxname != 'tlnews' and \
|
||
boxname != 'tlevents':
|
||
return
|
||
boxDir = createPersonDir(nickname, domain, baseDir, boxname)
|
||
for deleteFilename in os.scandir(boxDir):
|
||
deleteFilename = deleteFilename.name
|
||
filePath = os.path.join(boxDir, deleteFilename)
|
||
try:
|
||
if os.path.isfile(filePath):
|
||
os.unlink(filePath)
|
||
elif os.path.isdir(filePath):
|
||
shutil.rmtree(filePath)
|
||
except Exception as e:
|
||
print(e)
|
||
|
||
|
||
def savePostToBox(baseDir: str, httpPrefix: str, postId: str,
|
||
nickname: str, domain: str, postJsonObject: {},
|
||
boxname: str) -> str:
|
||
"""Saves the give json to the give box
|
||
Returns the filename
|
||
"""
|
||
if boxname != 'inbox' and boxname != 'outbox' and \
|
||
boxname != 'tlblogs' and boxname != 'tlnews' and \
|
||
boxname != 'tlevents' and \
|
||
boxname != 'scheduled':
|
||
return None
|
||
originalDomain = domain
|
||
if ':' in domain:
|
||
domain = domain.split(':')[0]
|
||
|
||
if not postId:
|
||
statusNumber, published = getStatusNumber()
|
||
postId = \
|
||
httpPrefix + '://' + originalDomain + '/users/' + nickname + \
|
||
'/statuses/' + statusNumber
|
||
postJsonObject['id'] = postId + '/activity'
|
||
if postJsonObject.get('object'):
|
||
if isinstance(postJsonObject['object'], dict):
|
||
postJsonObject['object']['id'] = postId
|
||
postJsonObject['object']['atomUri'] = postId
|
||
|
||
boxDir = createPersonDir(nickname, domain, baseDir, boxname)
|
||
filename = boxDir + '/' + postId.replace('/', '#') + '.json'
|
||
|
||
saveJson(postJsonObject, filename)
|
||
return filename
|
||
|
||
|
||
def updateHashtagsIndex(baseDir: str, tag: {}, newPostId: str) -> None:
|
||
"""Writes the post url for hashtags to a file
|
||
This allows posts for a hashtag to be quickly looked up
|
||
"""
|
||
if tag['type'] != 'Hashtag':
|
||
return
|
||
|
||
# create hashtags directory
|
||
tagsDir = baseDir + '/tags'
|
||
if not os.path.isdir(tagsDir):
|
||
os.mkdir(tagsDir)
|
||
tagName = tag['name']
|
||
tagsFilename = tagsDir + '/' + tagName[1:] + '.txt'
|
||
tagline = newPostId + '\n'
|
||
|
||
if not os.path.isfile(tagsFilename):
|
||
# create a new tags index file
|
||
tagsFile = open(tagsFilename, "w+")
|
||
if tagsFile:
|
||
tagsFile.write(tagline)
|
||
tagsFile.close()
|
||
else:
|
||
# prepend to tags index file
|
||
if tagline not in open(tagsFilename).read():
|
||
try:
|
||
with open(tagsFilename, 'r+') as tagsFile:
|
||
content = tagsFile.read()
|
||
tagsFile.seek(0, 0)
|
||
tagsFile.write(tagline+content)
|
||
except Exception as e:
|
||
print('WARN: Failed to write entry to tags file ' +
|
||
tagsFilename + ' ' + str(e))
|
||
|
||
|
||
def addSchedulePost(baseDir: str, nickname: str, domain: str,
|
||
eventDateStr: str, postId: str) -> None:
|
||
"""Adds a scheduled post to the index
|
||
"""
|
||
handle = nickname + '@' + domain
|
||
scheduleIndexFilename = baseDir + '/accounts/' + handle + '/schedule.index'
|
||
|
||
indexStr = eventDateStr + ' ' + postId.replace('/', '#')
|
||
if os.path.isfile(scheduleIndexFilename):
|
||
if indexStr not in open(scheduleIndexFilename).read():
|
||
try:
|
||
with open(scheduleIndexFilename, 'r+') as scheduleFile:
|
||
content = scheduleFile.read()
|
||
scheduleFile.seek(0, 0)
|
||
scheduleFile.write(indexStr + '\n' + content)
|
||
print('DEBUG: scheduled post added to index')
|
||
except Exception as e:
|
||
print('WARN: Failed to write entry to scheduled posts index ' +
|
||
scheduleIndexFilename + ' ' + str(e))
|
||
else:
|
||
scheduleFile = open(scheduleIndexFilename, 'w+')
|
||
if scheduleFile:
|
||
scheduleFile.write(indexStr + '\n')
|
||
scheduleFile.close()
|
||
|
||
|
||
def appendEventFields(newPost: {},
|
||
eventUUID: str, eventStatus: str,
|
||
anonymousParticipationEnabled: bool,
|
||
repliesModerationOption: str,
|
||
category: str,
|
||
joinMode: str,
|
||
eventDateStr: str,
|
||
endDateStr: str,
|
||
location: str,
|
||
maximumAttendeeCapacity: int,
|
||
ticketUrl: str,
|
||
subject: str) -> None:
|
||
"""Appends Mobilizon-type event fields to a post
|
||
"""
|
||
if not eventUUID:
|
||
return
|
||
|
||
# add attributes for Mobilizon-type events
|
||
newPost['uuid'] = eventUUID
|
||
if eventStatus:
|
||
newPost['ical:status'] = eventStatus
|
||
if anonymousParticipationEnabled:
|
||
newPost['anonymousParticipationEnabled'] = \
|
||
anonymousParticipationEnabled
|
||
if repliesModerationOption:
|
||
newPost['repliesModerationOption'] = repliesModerationOption
|
||
if category:
|
||
newPost['category'] = category
|
||
if joinMode:
|
||
newPost['joinMode'] = joinMode
|
||
newPost['startTime'] = eventDateStr
|
||
newPost['endTime'] = endDateStr
|
||
if location:
|
||
newPost['location'] = location
|
||
if maximumAttendeeCapacity:
|
||
newPost['maximumAttendeeCapacity'] = maximumAttendeeCapacity
|
||
if ticketUrl:
|
||
newPost['ticketUrl'] = ticketUrl
|
||
if subject:
|
||
newPost['name'] = subject
|
||
newPost['summary'] = None
|
||
newPost['sensitive'] = False
|
||
|
||
|
||
def validContentWarning(cw: str) -> str:
|
||
"""Returns a validated content warning
|
||
"""
|
||
cw = removeHtml(cw)
|
||
# hashtags within content warnings apparently cause a lot of trouble
|
||
# so remove them
|
||
if '#' in cw:
|
||
cw = cw.replace('#', '').replace(' ', ' ')
|
||
return cw
|
||
|
||
|
||
def loadAutoCW(baseDir: str, nickname: str, domain: str) -> []:
|
||
"""Loads automatic CWs file and returns a list containing
|
||
the lines of the file
|
||
"""
|
||
filename = baseDir + '/accounts/' + \
|
||
nickname + '@' + domain + '/autocw.txt'
|
||
if not os.path.isfile(filename):
|
||
return []
|
||
with open(filename, "r") as f:
|
||
return f.readlines()
|
||
return []
|
||
|
||
|
||
def addAutoCW(baseDir: str, nickname: str, domain: str,
|
||
subject: str, content: str) -> str:
|
||
"""Appends any automatic CW to the subject line
|
||
and returns the new subject line
|
||
"""
|
||
newSubject = subject
|
||
autoCWList = loadAutoCW(baseDir, nickname, domain)
|
||
for cwRule in autoCWList:
|
||
if '->' not in cwRule:
|
||
continue
|
||
match = cwRule.split('->')[0].strip()
|
||
if match not in content:
|
||
continue
|
||
cwStr = cwRule.split('->')[1].strip()
|
||
if newSubject:
|
||
if cwStr not in newSubject:
|
||
newSubject += ', ' + cwStr
|
||
else:
|
||
newSubject = cwStr
|
||
return newSubject
|
||
|
||
|
||
def createPostBase(baseDir: str, nickname: str, domain: str, port: int,
|
||
toUrl: str, ccUrl: str, httpPrefix: str, content: str,
|
||
followersOnly: bool, saveToFile: bool, clientToServer: bool,
|
||
commentsEnabled: bool,
|
||
attachImageFilename: str,
|
||
mediaType: str, imageDescription: str,
|
||
useBlurhash: bool, isModerationReport: bool,
|
||
isArticle: bool,
|
||
inReplyTo=None,
|
||
inReplyToAtomUri=None, subject=None, schedulePost=False,
|
||
eventDate=None, eventTime=None, location=None,
|
||
eventUUID=None, category=None, joinMode=None,
|
||
endDate=None, endTime=None,
|
||
maximumAttendeeCapacity=None,
|
||
repliesModerationOption=None,
|
||
anonymousParticipationEnabled=None,
|
||
eventStatus=None, ticketUrl=None) -> {}:
|
||
"""Creates a message
|
||
"""
|
||
subject = addAutoCW(baseDir, nickname, domain, subject, content)
|
||
|
||
if nickname != 'news':
|
||
mentionedRecipients = \
|
||
getMentionedPeople(baseDir, httpPrefix, content, domain, False)
|
||
else:
|
||
mentionedRecipients = ''
|
||
|
||
tags = []
|
||
hashtagsDict = {}
|
||
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domain = domain + ':' + str(port)
|
||
|
||
# add tags
|
||
if nickname != 'news':
|
||
content = \
|
||
addHtmlTags(baseDir, httpPrefix,
|
||
nickname, domain, content,
|
||
mentionedRecipients,
|
||
hashtagsDict, True)
|
||
|
||
# replace emoji with unicode
|
||
tags = []
|
||
for tagName, tag in hashtagsDict.items():
|
||
tags.append(tag)
|
||
# get list of tags
|
||
if nickname != 'news':
|
||
content = replaceEmojiFromTags(content, tags, 'content')
|
||
# remove replaced emoji
|
||
hashtagsDictCopy = hashtagsDict.copy()
|
||
for tagName, tag in hashtagsDictCopy.items():
|
||
if tag.get('name'):
|
||
if tag['name'].startswith(':'):
|
||
if tag['name'] not in content:
|
||
del hashtagsDict[tagName]
|
||
|
||
statusNumber, published = getStatusNumber()
|
||
newPostId = \
|
||
httpPrefix + '://' + domain + '/users/' + \
|
||
nickname + '/statuses/' + statusNumber
|
||
|
||
sensitive = False
|
||
summary = None
|
||
if subject:
|
||
summary = validContentWarning(subject)
|
||
sensitive = True
|
||
|
||
toRecipients = []
|
||
toCC = []
|
||
if toUrl:
|
||
if not isinstance(toUrl, str):
|
||
print('ERROR: toUrl is not a string')
|
||
return None
|
||
toRecipients = [toUrl]
|
||
|
||
# who to send to
|
||
if mentionedRecipients:
|
||
for mention in mentionedRecipients:
|
||
if mention not in toCC:
|
||
toCC.append(mention)
|
||
|
||
# create a list of hashtags
|
||
# Only posts which are #Public are searchable by hashtag
|
||
if hashtagsDict:
|
||
isPublic = False
|
||
for recipient in toRecipients:
|
||
if recipient.endswith('#Public'):
|
||
isPublic = True
|
||
break
|
||
for tagName, tag in hashtagsDict.items():
|
||
tags.append(tag)
|
||
if isPublic:
|
||
updateHashtagsIndex(baseDir, tag, newPostId)
|
||
print('Content tags: ' + str(tags))
|
||
|
||
if inReplyTo and not sensitive:
|
||
# locate the post which this is a reply to and check if
|
||
# it has a content warning. If it does then reproduce
|
||
# the same warning
|
||
replyPostFilename = \
|
||
locatePost(baseDir, nickname, domain, inReplyTo)
|
||
if replyPostFilename:
|
||
replyToJson = loadJson(replyPostFilename)
|
||
if replyToJson:
|
||
if replyToJson.get('object'):
|
||
if replyToJson['object'].get('sensitive'):
|
||
if replyToJson['object']['sensitive']:
|
||
sensitive = True
|
||
if replyToJson['object'].get('summary'):
|
||
summary = replyToJson['object']['summary']
|
||
|
||
# get the ending date and time
|
||
endDateStr = None
|
||
if endDate:
|
||
eventName = summary
|
||
if not eventName:
|
||
eventName = content
|
||
endDateStr = endDate
|
||
if endTime:
|
||
if endTime.endswith('Z'):
|
||
endDateStr = endDate + 'T' + endTime
|
||
else:
|
||
endDateStr = endDate + 'T' + endTime + \
|
||
':00' + strftime("%z", gmtime())
|
||
else:
|
||
endDateStr = endDate + 'T12:00:00Z'
|
||
|
||
# get the starting date and time
|
||
eventDateStr = None
|
||
if eventDate:
|
||
eventName = summary
|
||
if not eventName:
|
||
eventName = content
|
||
eventDateStr = eventDate
|
||
if eventTime:
|
||
if eventTime.endswith('Z'):
|
||
eventDateStr = eventDate + 'T' + eventTime
|
||
else:
|
||
eventDateStr = eventDate + 'T' + eventTime + \
|
||
':00' + strftime("%z", gmtime())
|
||
else:
|
||
eventDateStr = eventDate + 'T12:00:00Z'
|
||
if not endDateStr:
|
||
endDateStr = eventDateStr
|
||
if not schedulePost and not eventUUID:
|
||
tags.append({
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
"type": "Event",
|
||
"name": eventName,
|
||
"startTime": eventDateStr,
|
||
"endTime": endDateStr
|
||
})
|
||
if location and not eventUUID:
|
||
tags.append({
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
"type": "Place",
|
||
"name": location
|
||
})
|
||
|
||
postContext = [
|
||
'https://www.w3.org/ns/activitystreams',
|
||
{
|
||
'Hashtag': 'as:Hashtag',
|
||
'sensitive': 'as:sensitive',
|
||
'toot': 'http://joinmastodon.org/ns#',
|
||
'votersCount': 'toot:votersCount'
|
||
}
|
||
]
|
||
|
||
# make sure that CC doesn't also contain a To address
|
||
# eg. To: [ "https://mydomain/users/foo/followers" ]
|
||
# CC: [ "X", "Y", "https://mydomain/users/foo", "Z" ]
|
||
removeFromCC = []
|
||
for ccRecipient in toCC:
|
||
for sendToActor in toRecipients:
|
||
if ccRecipient in sendToActor and \
|
||
ccRecipient not in removeFromCC:
|
||
removeFromCC.append(ccRecipient)
|
||
break
|
||
for ccRemoval in removeFromCC:
|
||
toCC.remove(ccRemoval)
|
||
|
||
# the type of post to be made
|
||
postObjectType = 'Note'
|
||
if eventUUID:
|
||
postObjectType = 'Event'
|
||
|
||
# public posts with mentions should include the mentioned
|
||
# actors within CC
|
||
# "I think we don’t notify about mentions unless the person
|
||
# is also addressed in to/cc" -- gargron
|
||
if toRecipients and toCC and mentionedRecipients:
|
||
if len(toRecipients) == 1 and len(toCC) == 1:
|
||
if toRecipients[0].endswith('#Public') and \
|
||
toCC[0].endswith('/followers'):
|
||
toCC += mentionedRecipients
|
||
|
||
if not clientToServer:
|
||
actorUrl = httpPrefix + '://' + domain + '/users/' + nickname
|
||
|
||
idStr = \
|
||
httpPrefix + '://' + domain + '/users/' + nickname + \
|
||
'/statuses/' + statusNumber + '/replies'
|
||
newPost = {
|
||
'@context': postContext,
|
||
'id': newPostId + '/activity',
|
||
'type': 'Create',
|
||
'actor': actorUrl,
|
||
'published': published,
|
||
'to': toRecipients,
|
||
'cc': toCC,
|
||
'object': {
|
||
'id': newPostId,
|
||
'type': postObjectType,
|
||
'summary': summary,
|
||
'inReplyTo': inReplyTo,
|
||
'published': published,
|
||
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
|
||
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
|
||
'to': toRecipients,
|
||
'cc': toCC,
|
||
'sensitive': sensitive,
|
||
'atomUri': newPostId,
|
||
'inReplyToAtomUri': inReplyToAtomUri,
|
||
'commentsEnabled': commentsEnabled,
|
||
'mediaType': 'text/html',
|
||
'content': content,
|
||
'contentMap': {
|
||
'en': content
|
||
},
|
||
'attachment': [],
|
||
'tag': tags,
|
||
'replies': {
|
||
'id': idStr,
|
||
'type': 'Collection',
|
||
'first': {
|
||
'type': 'CollectionPage',
|
||
'partOf': idStr,
|
||
'items': []
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if attachImageFilename:
|
||
newPost['object'] = \
|
||
attachMedia(baseDir, httpPrefix, domain, port,
|
||
newPost['object'], attachImageFilename,
|
||
mediaType, imageDescription, useBlurhash)
|
||
appendEventFields(newPost['object'], eventUUID, eventStatus,
|
||
anonymousParticipationEnabled,
|
||
repliesModerationOption,
|
||
category, joinMode,
|
||
eventDateStr, endDateStr,
|
||
location, maximumAttendeeCapacity,
|
||
ticketUrl, subject)
|
||
else:
|
||
idStr = \
|
||
httpPrefix + '://' + domain + '/users/' + nickname + \
|
||
'/statuses/' + statusNumber + '/replies'
|
||
newPost = {
|
||
"@context": postContext,
|
||
'id': newPostId,
|
||
'type': postObjectType,
|
||
'summary': summary,
|
||
'inReplyTo': inReplyTo,
|
||
'published': published,
|
||
'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
|
||
'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
|
||
'to': toRecipients,
|
||
'cc': toCC,
|
||
'sensitive': sensitive,
|
||
'atomUri': newPostId,
|
||
'inReplyToAtomUri': inReplyToAtomUri,
|
||
'commentsEnabled': commentsEnabled,
|
||
'mediaType': 'text/html',
|
||
'content': content,
|
||
'contentMap': {
|
||
'en': content
|
||
},
|
||
'attachment': [],
|
||
'tag': tags,
|
||
'replies': {
|
||
'id': idStr,
|
||
'type': 'Collection',
|
||
'first': {
|
||
'type': 'CollectionPage',
|
||
'partOf': idStr,
|
||
'items': []
|
||
}
|
||
}
|
||
}
|
||
if attachImageFilename:
|
||
newPost = \
|
||
attachMedia(baseDir, httpPrefix, domain, port,
|
||
newPost, attachImageFilename,
|
||
mediaType, imageDescription, useBlurhash)
|
||
appendEventFields(newPost, eventUUID, eventStatus,
|
||
anonymousParticipationEnabled,
|
||
repliesModerationOption,
|
||
category, joinMode,
|
||
eventDateStr, endDateStr,
|
||
location, maximumAttendeeCapacity,
|
||
ticketUrl, subject)
|
||
if ccUrl:
|
||
if len(ccUrl) > 0:
|
||
newPost['cc'] = [ccUrl]
|
||
if newPost.get('object'):
|
||
newPost['object']['cc'] = [ccUrl]
|
||
|
||
# if this is a moderation report then add a status
|
||
if isModerationReport:
|
||
# add status
|
||
if newPost.get('object'):
|
||
newPost['object']['moderationStatus'] = 'pending'
|
||
else:
|
||
newPost['moderationStatus'] = 'pending'
|
||
# save to index file
|
||
moderationIndexFile = baseDir + '/accounts/moderation.txt'
|
||
modFile = open(moderationIndexFile, "a+")
|
||
if modFile:
|
||
modFile.write(newPostId + '\n')
|
||
modFile.close()
|
||
|
||
# If a patch has been posted - i.e. the output from
|
||
# git format-patch - then convert the activitypub type
|
||
convertPostToPatch(baseDir, nickname, domain, newPost)
|
||
|
||
if schedulePost:
|
||
if eventDate and eventTime:
|
||
# add an item to the scheduled post index file
|
||
addSchedulePost(baseDir, nickname, domain, eventDateStr, newPostId)
|
||
savePostToBox(baseDir, httpPrefix, newPostId,
|
||
nickname, domain, newPost, 'scheduled')
|
||
else:
|
||
print('Unable to create scheduled post without ' +
|
||
'date and time values')
|
||
return newPost
|
||
elif saveToFile:
|
||
if isArticle:
|
||
savePostToBox(baseDir, httpPrefix, newPostId,
|
||
nickname, domain, newPost, 'tlblogs')
|
||
elif eventUUID:
|
||
savePostToBox(baseDir, httpPrefix, newPostId,
|
||
nickname, domain, newPost, 'tlevents')
|
||
else:
|
||
savePostToBox(baseDir, httpPrefix, newPostId,
|
||
nickname, domain, newPost, 'outbox')
|
||
return newPost
|
||
|
||
|
||
def outboxMessageCreateWrap(httpPrefix: str,
|
||
nickname: str, domain: str, port: int,
|
||
messageJson: {}) -> {}:
|
||
"""Wraps a received message in a Create
|
||
https://www.w3.org/TR/activitypub/#object-without-create
|
||
"""
|
||
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domain = domain + ':' + str(port)
|
||
statusNumber, published = getStatusNumber()
|
||
if messageJson.get('published'):
|
||
published = messageJson['published']
|
||
newPostId = \
|
||
httpPrefix + '://' + domain + '/users/' + nickname + \
|
||
'/statuses/' + statusNumber
|
||
cc = []
|
||
if messageJson.get('cc'):
|
||
cc = messageJson['cc']
|
||
newPost = {
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
'id': newPostId + '/activity',
|
||
'type': 'Create',
|
||
'actor': httpPrefix + '://' + domain + '/users/' + nickname,
|
||
'published': published,
|
||
'to': messageJson['to'],
|
||
'cc': cc,
|
||
'object': messageJson
|
||
}
|
||
newPost['object']['id'] = newPost['id']
|
||
newPost['object']['url'] = \
|
||
httpPrefix + '://' + domain + '/@' + nickname + '/' + statusNumber
|
||
newPost['object']['atomUri'] = \
|
||
httpPrefix + '://' + domain + '/users/' + nickname + \
|
||
'/statuses/' + statusNumber
|
||
return newPost
|
||
|
||
|
||
def postIsAddressedToFollowers(baseDir: str,
|
||
nickname: str, domain: str, port: int,
|
||
httpPrefix: str,
|
||
postJsonObject: {}) -> bool:
|
||
"""Returns true if the given post is addressed to followers of the nickname
|
||
"""
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domain = domain + ':' + str(port)
|
||
|
||
if not postJsonObject.get('object'):
|
||
return False
|
||
toList = []
|
||
ccList = []
|
||
if postJsonObject['type'] != 'Update' and \
|
||
isinstance(postJsonObject['object'], dict):
|
||
if postJsonObject['object'].get('to'):
|
||
toList = postJsonObject['object']['to']
|
||
if postJsonObject['object'].get('cc'):
|
||
ccList = postJsonObject['object']['cc']
|
||
else:
|
||
if postJsonObject.get('to'):
|
||
toList = postJsonObject['to']
|
||
if postJsonObject.get('cc'):
|
||
ccList = postJsonObject['cc']
|
||
|
||
followersUrl = httpPrefix + '://' + domain + '/users/' + \
|
||
nickname + '/followers'
|
||
|
||
# does the followers url exist in 'to' or 'cc' lists?
|
||
addressedToFollowers = False
|
||
if followersUrl in toList:
|
||
addressedToFollowers = True
|
||
elif followersUrl in ccList:
|
||
addressedToFollowers = True
|
||
return addressedToFollowers
|
||
|
||
|
||
def postIsAddressedToPublic(baseDir: str, postJsonObject: {}) -> bool:
|
||
"""Returns true if the given post is addressed to public
|
||
"""
|
||
if not postJsonObject.get('object'):
|
||
return False
|
||
if not postJsonObject['object'].get('to'):
|
||
return False
|
||
|
||
publicUrl = 'https://www.w3.org/ns/activitystreams#Public'
|
||
|
||
# does the public url exist in 'to' or 'cc' lists?
|
||
addressedToPublic = False
|
||
if publicUrl in postJsonObject['object']['to']:
|
||
addressedToPublic = True
|
||
if not addressedToPublic:
|
||
if not postJsonObject['object'].get('cc'):
|
||
return False
|
||
if publicUrl in postJsonObject['object']['cc']:
|
||
addressedToPublic = True
|
||
return addressedToPublic
|
||
|
||
|
||
def createPublicPost(baseDir: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
content: str, followersOnly: bool, saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
inReplyTo=None, inReplyToAtomUri=None, subject=None,
|
||
schedulePost=False,
|
||
eventDate=None, eventTime=None, location=None) -> {}:
|
||
"""Public post
|
||
"""
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
return createPostBase(baseDir, nickname, domain, port,
|
||
'https://www.w3.org/ns/activitystreams#Public',
|
||
httpPrefix + '://' + domainFull + '/users/' +
|
||
nickname + '/followers',
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost, eventDate, eventTime, location,
|
||
None, None, None, None, None,
|
||
None, None, None, None, None)
|
||
|
||
|
||
def createBlogPost(baseDir: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
content: str, followersOnly: bool, saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
inReplyTo=None, inReplyToAtomUri=None, subject=None,
|
||
schedulePost=False,
|
||
eventDate=None, eventTime=None, location=None) -> {}:
|
||
blog = \
|
||
createPublicPost(baseDir,
|
||
nickname, domain, port, httpPrefix,
|
||
content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost,
|
||
eventDate, eventTime, location)
|
||
blog['object']['type'] = 'Article'
|
||
|
||
# append citations tags, stored in a file
|
||
citationsFilename = \
|
||
baseDir + '/accounts/' + \
|
||
nickname + '@' + domain + '/.citations.txt'
|
||
if os.path.isfile(citationsFilename):
|
||
citationsSeparator = '#####'
|
||
with open(citationsFilename, "r") as f:
|
||
citations = f.readlines()
|
||
for line in citations:
|
||
if citationsSeparator not in line:
|
||
continue
|
||
sections = line.strip().split(citationsSeparator)
|
||
if len(sections) != 3:
|
||
continue
|
||
# dateStr = sections[0]
|
||
title = sections[1]
|
||
link = sections[2]
|
||
tagJson = {
|
||
"type": "Article",
|
||
"name": title,
|
||
"url": link
|
||
}
|
||
blog['object']['tag'].append(tagJson)
|
||
|
||
return blog
|
||
|
||
|
||
def createNewsPost(baseDir: str,
|
||
domain: str, port: int, httpPrefix: str,
|
||
content: str, followersOnly: bool, saveToFile: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
subject: str) -> {}:
|
||
clientToServer = False
|
||
inReplyTo = None
|
||
inReplyToAtomUri = None
|
||
schedulePost = False
|
||
eventDate = None
|
||
eventTime = None
|
||
location = None
|
||
blog = \
|
||
createPublicPost(baseDir,
|
||
'news', domain, port, httpPrefix,
|
||
content, followersOnly, saveToFile,
|
||
clientToServer, False,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost,
|
||
eventDate, eventTime, location)
|
||
blog['object']['type'] = 'Article'
|
||
return blog
|
||
|
||
|
||
def createQuestionPost(baseDir: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
content: str, qOptions: [],
|
||
followersOnly: bool, saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
subject: str, durationDays: int) -> {}:
|
||
"""Question post with multiple choice options
|
||
"""
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
messageJson = \
|
||
createPostBase(baseDir, nickname, domain, port,
|
||
'https://www.w3.org/ns/activitystreams#Public',
|
||
httpPrefix + '://' + domainFull + '/users/' +
|
||
nickname + '/followers',
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, None, None, subject,
|
||
False, None, None, None, None, None,
|
||
None, None, None,
|
||
None, None, None, None, None)
|
||
messageJson['object']['type'] = 'Question'
|
||
messageJson['object']['oneOf'] = []
|
||
messageJson['object']['votersCount'] = 0
|
||
currTime = datetime.datetime.utcnow()
|
||
daysSinceEpoch = \
|
||
int((currTime - datetime.datetime(1970, 1, 1)).days + durationDays)
|
||
endTime = datetime.datetime(1970, 1, 1) + \
|
||
datetime.timedelta(daysSinceEpoch)
|
||
messageJson['object']['endTime'] = endTime.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
for questionOption in qOptions:
|
||
messageJson['object']['oneOf'].append({
|
||
"type": "Note",
|
||
"name": questionOption,
|
||
"replies": {
|
||
"type": "Collection",
|
||
"totalItems": 0
|
||
}
|
||
})
|
||
return messageJson
|
||
|
||
|
||
def createUnlistedPost(baseDir: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
content: str, followersOnly: bool, saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
inReplyTo=None, inReplyToAtomUri=None, subject=None,
|
||
schedulePost=False,
|
||
eventDate=None, eventTime=None, location=None) -> {}:
|
||
"""Unlisted post. This has the #Public and followers links inverted.
|
||
"""
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
return createPostBase(baseDir, nickname, domain, port,
|
||
httpPrefix + '://' + domainFull + '/users/' +
|
||
nickname + '/followers',
|
||
'https://www.w3.org/ns/activitystreams#Public',
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost, eventDate, eventTime, location,
|
||
None, None, None, None, None,
|
||
None, None, None, None, None)
|
||
|
||
|
||
def createFollowersOnlyPost(baseDir: str,
|
||
nickname: str, domain: str, port: int,
|
||
httpPrefix: str,
|
||
content: str, followersOnly: bool,
|
||
saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
inReplyTo=None, inReplyToAtomUri=None,
|
||
subject=None, schedulePost=False,
|
||
eventDate=None, eventTime=None,
|
||
location=None) -> {}:
|
||
"""Followers only post
|
||
"""
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
return createPostBase(baseDir, nickname, domain, port,
|
||
httpPrefix + '://' + domainFull + '/users/' +
|
||
nickname + '/followers',
|
||
None,
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost, eventDate, eventTime, location,
|
||
None, None, None, None, None,
|
||
None, None, None, None, None)
|
||
|
||
|
||
def createEventPost(baseDir: str,
|
||
nickname: str, domain: str, port: int,
|
||
httpPrefix: str,
|
||
content: str, followersOnly: bool,
|
||
saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
subject=None, schedulePost=False,
|
||
eventDate=None, eventTime=None,
|
||
location=None, category=None, joinMode=None,
|
||
endDate=None, endTime=None,
|
||
maximumAttendeeCapacity=None,
|
||
repliesModerationOption=None,
|
||
anonymousParticipationEnabled=None,
|
||
eventStatus=None, ticketUrl=None) -> {}:
|
||
"""Mobilizon-type Event post
|
||
"""
|
||
if not attachImageFilename:
|
||
print('Event has no attached image')
|
||
return None
|
||
if not category:
|
||
print('Event has no category')
|
||
return None
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
|
||
# create event uuid
|
||
eventUUID = str(uuid.uuid1())
|
||
|
||
toStr1 = 'https://www.w3.org/ns/activitystreams#Public'
|
||
toStr2 = httpPrefix + '://' + domainFull + '/users/' + \
|
||
nickname + '/followers',
|
||
if followersOnly:
|
||
toStr1 = toStr2
|
||
toStr2 = None
|
||
return createPostBase(baseDir, nickname, domain, port,
|
||
toStr1, toStr2,
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, None, None, subject,
|
||
schedulePost, eventDate, eventTime, location,
|
||
eventUUID, category, joinMode,
|
||
endDate, endTime, maximumAttendeeCapacity,
|
||
repliesModerationOption,
|
||
anonymousParticipationEnabled,
|
||
eventStatus, ticketUrl)
|
||
|
||
|
||
def getMentionedPeople(baseDir: str, httpPrefix: str,
|
||
content: str, domain: str, debug: bool) -> []:
|
||
"""Extracts a list of mentioned actors from the given message content
|
||
"""
|
||
if '@' not in content:
|
||
return None
|
||
mentions = []
|
||
words = content.split(' ')
|
||
for wrd in words:
|
||
if wrd.startswith('@'):
|
||
handle = wrd[1:]
|
||
if debug:
|
||
print('DEBUG: mentioned handle ' + handle)
|
||
if '@' not in handle:
|
||
handle = handle + '@' + domain
|
||
if not os.path.isdir(baseDir + '/accounts/' + handle):
|
||
continue
|
||
else:
|
||
externalDomain = handle.split('@')[1]
|
||
if not ('.' in externalDomain or
|
||
externalDomain == 'localhost'):
|
||
continue
|
||
mentionedNickname = handle.split('@')[0]
|
||
mentionedDomain = handle.split('@')[1].strip('\n').strip('\r')
|
||
if ':' in mentionedDomain:
|
||
mentionedDomain = mentionedDomain.split(':')[0]
|
||
if not validNickname(mentionedDomain, mentionedNickname):
|
||
continue
|
||
actor = \
|
||
httpPrefix + '://' + handle.split('@')[1] + \
|
||
'/users/' + mentionedNickname
|
||
mentions.append(actor)
|
||
return mentions
|
||
|
||
|
||
def createDirectMessagePost(baseDir: str,
|
||
nickname: str, domain: str, port: int,
|
||
httpPrefix: str,
|
||
content: str, followersOnly: bool,
|
||
saveToFile: bool, clientToServer: bool,
|
||
commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
inReplyTo=None, inReplyToAtomUri=None,
|
||
subject=None, debug=False,
|
||
schedulePost=False,
|
||
eventDate=None, eventTime=None,
|
||
location=None) -> {}:
|
||
"""Direct Message post
|
||
"""
|
||
content = resolvePetnames(baseDir, nickname, domain, content)
|
||
mentionedPeople = \
|
||
getMentionedPeople(baseDir, httpPrefix, content, domain, debug)
|
||
if debug:
|
||
print('mentionedPeople: ' + str(mentionedPeople))
|
||
if not mentionedPeople:
|
||
return None
|
||
postTo = None
|
||
postCc = None
|
||
messageJson = \
|
||
createPostBase(baseDir, nickname, domain, port,
|
||
postTo, postCc,
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, False, inReplyTo, inReplyToAtomUri, subject,
|
||
schedulePost, eventDate, eventTime, location,
|
||
None, None, None, None, None,
|
||
None, None, None, None, None)
|
||
# mentioned recipients go into To rather than Cc
|
||
messageJson['to'] = messageJson['object']['cc']
|
||
messageJson['object']['to'] = messageJson['to']
|
||
messageJson['cc'] = []
|
||
messageJson['object']['cc'] = []
|
||
if schedulePost:
|
||
savePostToBox(baseDir, httpPrefix, messageJson['object']['id'],
|
||
nickname, domain, messageJson, 'scheduled')
|
||
return messageJson
|
||
|
||
|
||
def createReportPost(baseDir: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
content: str, followersOnly: bool, saveToFile: bool,
|
||
clientToServer: bool, commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
debug: bool, subject=None) -> {}:
|
||
"""Send a report to moderators
|
||
"""
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
|
||
# add a title to distinguish moderation reports from other posts
|
||
reportTitle = 'Moderation Report'
|
||
if not subject:
|
||
subject = reportTitle
|
||
else:
|
||
if not subject.startswith(reportTitle):
|
||
subject = reportTitle + ': ' + subject
|
||
|
||
# create the list of moderators from the moderators file
|
||
moderatorsList = []
|
||
moderatorsFile = baseDir + '/accounts/moderators.txt'
|
||
if os.path.isfile(moderatorsFile):
|
||
with open(moderatorsFile, "r") as fileHandler:
|
||
for line in fileHandler:
|
||
line = line.strip('\n').strip('\r')
|
||
if line.startswith('#'):
|
||
continue
|
||
if line.startswith('/users/'):
|
||
line = line.replace('users', '')
|
||
if line.startswith('@'):
|
||
line = line[1:]
|
||
if '@' in line:
|
||
moderatorActor = httpPrefix + '://' + domainFull + \
|
||
'/users/' + line.split('@')[0]
|
||
if moderatorActor not in moderatorsList:
|
||
moderatorsList.append(moderatorActor)
|
||
continue
|
||
if line.startswith('http') or line.startswith('dat'):
|
||
# must be a local address - no remote moderators
|
||
if '://' + domainFull + '/' in line:
|
||
if line not in moderatorsList:
|
||
moderatorsList.append(line)
|
||
else:
|
||
if '/' not in line:
|
||
moderatorActor = httpPrefix + '://' + domainFull + \
|
||
'/users/' + line
|
||
if moderatorActor not in moderatorsList:
|
||
moderatorsList.append(moderatorActor)
|
||
if len(moderatorsList) == 0:
|
||
# if there are no moderators then the admin becomes the moderator
|
||
adminNickname = getConfigParam(baseDir, 'admin')
|
||
if adminNickname:
|
||
moderatorsList.append(httpPrefix + '://' + domainFull +
|
||
'/users/' + adminNickname)
|
||
if not moderatorsList:
|
||
return None
|
||
if debug:
|
||
print('DEBUG: Sending report to moderators')
|
||
print(str(moderatorsList))
|
||
postTo = moderatorsList
|
||
postCc = None
|
||
postJsonObject = None
|
||
for toUrl in postTo:
|
||
# who is this report going to?
|
||
toNickname = toUrl.split('/users/')[1]
|
||
handle = toNickname + '@' + domain
|
||
|
||
postJsonObject = \
|
||
createPostBase(baseDir, nickname, domain, port,
|
||
toUrl, postCc,
|
||
httpPrefix, content, followersOnly, saveToFile,
|
||
clientToServer, commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
True, False, None, None, subject,
|
||
False, None, None, None, None, None,
|
||
None, None, None,
|
||
None, None, None, None, None)
|
||
if not postJsonObject:
|
||
continue
|
||
|
||
# update the inbox index with the report filename
|
||
# indexFilename = baseDir+'/accounts/'+handle+'/inbox.index'
|
||
# indexEntry = \
|
||
# removeIdEnding(postJsonObject['id']).replace('/','#') + '.json'
|
||
# if indexEntry not in open(indexFilename).read():
|
||
# try:
|
||
# with open(indexFilename, 'a+') as fp:
|
||
# fp.write(indexEntry)
|
||
# except:
|
||
# pass
|
||
|
||
# save a notification file so that the moderator
|
||
# knows something new has appeared
|
||
newReportFile = baseDir + '/accounts/' + handle + '/.newReport'
|
||
if os.path.isfile(newReportFile):
|
||
continue
|
||
try:
|
||
with open(newReportFile, 'w+') as fp:
|
||
fp.write(toUrl + '/moderation')
|
||
except BaseException:
|
||
pass
|
||
|
||
return postJsonObject
|
||
|
||
|
||
def threadSendPost(session, postJsonStr: str, federationList: [],
|
||
inboxUrl: str, baseDir: str,
|
||
signatureHeaderJson: {}, postLog: [],
|
||
debug: bool) -> None:
|
||
"""Sends a with retries
|
||
"""
|
||
tries = 0
|
||
sendIntervalSec = 30
|
||
for attempt in range(20):
|
||
postResult = None
|
||
unauthorized = False
|
||
try:
|
||
postResult, unauthorized = \
|
||
postJsonString(session, postJsonStr, federationList,
|
||
inboxUrl, signatureHeaderJson,
|
||
debug)
|
||
except Exception as e:
|
||
print('ERROR: postJsonString failed ' + str(e))
|
||
if unauthorized:
|
||
print(postJsonStr)
|
||
print('threadSendPost: Post is unauthorized')
|
||
break
|
||
if postResult:
|
||
logStr = 'Success on try ' + str(tries) + ': ' + postJsonStr
|
||
else:
|
||
logStr = 'Retry ' + str(tries) + ': ' + postJsonStr
|
||
postLog.append(logStr)
|
||
# keep the length of the log finite
|
||
# Don't accumulate massive files on systems with limited resources
|
||
while len(postLog) > 16:
|
||
postLog.pop(0)
|
||
if debug:
|
||
# save the log file
|
||
postLogFilename = baseDir + '/post.log'
|
||
with open(postLogFilename, "a+") as logFile:
|
||
logFile.write(logStr + '\n')
|
||
|
||
if postResult:
|
||
if debug:
|
||
print('DEBUG: successful json post to ' + inboxUrl)
|
||
# our work here is done
|
||
break
|
||
if debug:
|
||
print(postJsonStr)
|
||
print('DEBUG: json post to ' + inboxUrl +
|
||
' failed. Waiting for ' +
|
||
str(sendIntervalSec) + ' seconds.')
|
||
time.sleep(sendIntervalSec)
|
||
tries += 1
|
||
|
||
|
||
def sendPost(projectVersion: str,
|
||
session, baseDir: str, nickname: str, domain: str, port: int,
|
||
toNickname: str, toDomain: str, toPort: int, cc: str,
|
||
httpPrefix: str, content: str, followersOnly: bool,
|
||
saveToFile: bool, clientToServer: bool,
|
||
commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
federationList: [], sendThreads: [], postLog: [],
|
||
cachedWebfingers: {}, personCache: {},
|
||
isArticle: bool,
|
||
debug=False, inReplyTo=None,
|
||
inReplyToAtomUri=None, subject=None) -> int:
|
||
"""Post to another inbox
|
||
"""
|
||
withDigest = True
|
||
|
||
if toNickname == 'inbox':
|
||
# shared inbox actor on @domain@domain
|
||
toNickname = toDomain
|
||
|
||
if toPort:
|
||
if toPort != 80 and toPort != 443:
|
||
if ':' not in toDomain:
|
||
toDomain = toDomain + ':' + str(toPort)
|
||
|
||
handle = httpPrefix + '://' + toDomain + '/@' + toNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = webfingerHandle(session, handle, httpPrefix,
|
||
cachedWebfingers,
|
||
domain, projectVersion)
|
||
if not wfRequest:
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return 1
|
||
|
||
if not clientToServer:
|
||
postToBox = 'inbox'
|
||
else:
|
||
postToBox = 'outbox'
|
||
if isArticle:
|
||
postToBox = 'tlblogs'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
toPersonId, sharedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix,
|
||
nickname, domain, postToBox)
|
||
|
||
if not inboxUrl:
|
||
return 3
|
||
if not pubKey:
|
||
return 4
|
||
if not toPersonId:
|
||
return 5
|
||
# sharedInbox is optional
|
||
|
||
postJsonObject = \
|
||
createPostBase(baseDir, nickname, domain, port,
|
||
toPersonId, cc, httpPrefix, content,
|
||
followersOnly, saveToFile, clientToServer,
|
||
commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, isArticle, inReplyTo,
|
||
inReplyToAtomUri, subject,
|
||
False, None, None, None, None, None,
|
||
None, None, None,
|
||
None, None, None, None, None)
|
||
|
||
# get the senders private key
|
||
privateKeyPem = getPersonKey(nickname, domain, baseDir, 'private')
|
||
if len(privateKeyPem) == 0:
|
||
return 6
|
||
|
||
if toDomain not in inboxUrl:
|
||
return 7
|
||
postPath = inboxUrl.split(toDomain, 1)[1]
|
||
|
||
if not postJsonObject.get('signature'):
|
||
try:
|
||
signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem)
|
||
postJsonObject = signedPostJsonObject
|
||
except Exception as e:
|
||
print('WARN: failed to JSON-LD sign post, ' + str(e))
|
||
pass
|
||
|
||
# convert json to string so that there are no
|
||
# subsequent conversions after creating message body digest
|
||
postJsonStr = json.dumps(postJsonObject)
|
||
|
||
# construct the http header, including the message body digest
|
||
signatureHeaderJson = \
|
||
createSignedHeader(privateKeyPem, nickname, domain, port,
|
||
toDomain, toPort,
|
||
postPath, httpPrefix, withDigest, postJsonStr)
|
||
|
||
# Keep the number of threads being used small
|
||
while len(sendThreads) > 1000:
|
||
print('WARN: Maximum threads reached - killing send thread')
|
||
sendThreads[0].kill()
|
||
sendThreads.pop(0)
|
||
print('WARN: thread killed')
|
||
thr = \
|
||
threadWithTrace(target=threadSendPost,
|
||
args=(session,
|
||
postJsonStr,
|
||
federationList,
|
||
inboxUrl, baseDir,
|
||
signatureHeaderJson.copy(),
|
||
postLog,
|
||
debug), daemon=True)
|
||
sendThreads.append(thr)
|
||
thr.start()
|
||
return 0
|
||
|
||
|
||
def sendPostViaServer(projectVersion: str,
|
||
baseDir: str, session, fromNickname: str, password: str,
|
||
fromDomain: str, fromPort: int,
|
||
toNickname: str, toDomain: str, toPort: int, cc: str,
|
||
httpPrefix: str, content: str, followersOnly: bool,
|
||
commentsEnabled: bool,
|
||
attachImageFilename: str, mediaType: str,
|
||
imageDescription: str, useBlurhash: bool,
|
||
cachedWebfingers: {}, personCache: {},
|
||
isArticle: bool, debug=False, inReplyTo=None,
|
||
inReplyToAtomUri=None, subject=None) -> int:
|
||
"""Send a post via a proxy (c2s)
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendPostViaServer')
|
||
return 6
|
||
|
||
if toPort:
|
||
if toPort != 80 and toPort != 443:
|
||
if ':' not in fromDomain:
|
||
fromDomain = fromDomain + ':' + str(fromPort)
|
||
|
||
handle = httpPrefix + '://' + fromDomain + '/@' + fromNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = \
|
||
webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
|
||
fromDomain, projectVersion)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: webfinger failed for ' + handle)
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return 1
|
||
|
||
postToBox = 'outbox'
|
||
if isArticle:
|
||
postToBox = 'tlblogs'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
fromPersonId, sharedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix,
|
||
fromNickname,
|
||
fromDomain, postToBox)
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: No ' + postToBox + ' was found for ' + handle)
|
||
return 3
|
||
if not fromPersonId:
|
||
if debug:
|
||
print('DEBUG: No actor was found for ' + handle)
|
||
return 4
|
||
|
||
# Get the json for the c2s post, not saving anything to file
|
||
# Note that baseDir is set to None
|
||
saveToFile = False
|
||
clientToServer = True
|
||
if toDomain.lower().endswith('public'):
|
||
toPersonId = 'https://www.w3.org/ns/activitystreams#Public'
|
||
fromDomainFull = fromDomain
|
||
if fromPort:
|
||
if fromPort != 80 and fromPort != 443:
|
||
if ':' not in fromDomain:
|
||
fromDomainFull = fromDomain + ':' + str(fromPort)
|
||
cc = httpPrefix + '://' + fromDomainFull + '/users/' + \
|
||
fromNickname + '/followers'
|
||
else:
|
||
if toDomain.lower().endswith('followers') or \
|
||
toDomain.lower().endswith('followersonly'):
|
||
toPersonId = \
|
||
httpPrefix + '://' + \
|
||
fromDomainFull + '/users/' + fromNickname + '/followers'
|
||
else:
|
||
toDomainFull = toDomain
|
||
if toPort:
|
||
if toPort != 80 and toPort != 443:
|
||
if ':' not in toDomain:
|
||
toDomainFull = toDomain + ':' + str(toPort)
|
||
toPersonId = httpPrefix + '://' + toDomainFull + \
|
||
'/users/' + toNickname
|
||
|
||
postJsonObject = \
|
||
createPostBase(baseDir,
|
||
fromNickname, fromDomain, fromPort,
|
||
toPersonId, cc, httpPrefix, content,
|
||
followersOnly, saveToFile, clientToServer,
|
||
commentsEnabled,
|
||
attachImageFilename, mediaType,
|
||
imageDescription, useBlurhash,
|
||
False, isArticle, inReplyTo,
|
||
inReplyToAtomUri, subject,
|
||
False, None, None, None, None, None,
|
||
None, None, None,
|
||
None, None, None, None, None)
|
||
|
||
authHeader = createBasicAuthHeader(fromNickname, password)
|
||
|
||
if attachImageFilename:
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = \
|
||
postImage(session, attachImageFilename, [],
|
||
inboxUrl, headers)
|
||
if not postResult:
|
||
if debug:
|
||
print('DEBUG: Failed to upload image')
|
||
# return 9
|
||
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = \
|
||
postJsonString(session, json.dumps(postJsonObject), [],
|
||
inboxUrl, headers, debug)
|
||
if not postResult:
|
||
if debug:
|
||
print('DEBUG: POST failed for c2s to '+inboxUrl)
|
||
return 5
|
||
|
||
if debug:
|
||
print('DEBUG: c2s POST success')
|
||
return 0
|
||
|
||
|
||
def groupFollowersByDomain(baseDir: str, nickname: str, domain: str) -> {}:
|
||
"""Returns a dictionary with followers grouped by domain
|
||
"""
|
||
handle = nickname + '@' + domain
|
||
followersFilename = baseDir + '/accounts/' + handle + '/followers.txt'
|
||
if not os.path.isfile(followersFilename):
|
||
return None
|
||
grouped = {}
|
||
with open(followersFilename, "r") as f:
|
||
for followerHandle in f:
|
||
if '@' in followerHandle:
|
||
fHandle = \
|
||
followerHandle.strip().replace('\n', '').replace('\r', '')
|
||
followerDomain = fHandle.split('@')[1]
|
||
if not grouped.get(followerDomain):
|
||
grouped[followerDomain] = [fHandle]
|
||
else:
|
||
grouped[followerDomain].append(fHandle)
|
||
return grouped
|
||
|
||
|
||
def addFollowersToPublicPost(postJsonObject: {}) -> None:
|
||
"""Adds followers entry to cc if it doesn't exist
|
||
"""
|
||
if not postJsonObject.get('actor'):
|
||
return
|
||
|
||
if isinstance(postJsonObject['object'], str):
|
||
if not postJsonObject.get('to'):
|
||
return
|
||
if len(postJsonObject['to']) > 1:
|
||
return
|
||
if len(postJsonObject['to']) == 0:
|
||
return
|
||
if not postJsonObject['to'][0].endswith('#Public'):
|
||
return
|
||
if postJsonObject.get('cc'):
|
||
return
|
||
postJsonObject['cc'] = postJsonObject['actor'] + '/followers'
|
||
elif isinstance(postJsonObject['object'], dict):
|
||
if not postJsonObject['object'].get('to'):
|
||
return
|
||
if len(postJsonObject['object']['to']) > 1:
|
||
return
|
||
if len(postJsonObject['object']['to']) == 0:
|
||
return
|
||
if not postJsonObject['object']['to'][0].endswith('#Public'):
|
||
return
|
||
if postJsonObject['object'].get('cc'):
|
||
return
|
||
postJsonObject['object']['cc'] = postJsonObject['actor'] + '/followers'
|
||
|
||
|
||
def sendSignedJson(postJsonObject: {}, session, baseDir: str,
|
||
nickname: str, domain: str, port: int,
|
||
toNickname: str, toDomain: str, toPort: int, cc: str,
|
||
httpPrefix: str, saveToFile: bool, clientToServer: bool,
|
||
federationList: [],
|
||
sendThreads: [], postLog: [], cachedWebfingers: {},
|
||
personCache: {}, debug: bool, projectVersion: str) -> int:
|
||
"""Sends a signed json object to an inbox/outbox
|
||
"""
|
||
if debug:
|
||
print('DEBUG: sendSignedJson start')
|
||
if not session:
|
||
print('WARN: No session specified for sendSignedJson')
|
||
return 8
|
||
withDigest = True
|
||
|
||
if toDomain.endswith('.onion') or toDomain.endswith('.i2p'):
|
||
httpPrefix = 'http'
|
||
|
||
# sharedInbox = False
|
||
if toNickname == 'inbox':
|
||
# shared inbox actor on @domain@domain
|
||
toNickname = toDomain
|
||
# sharedInbox = True
|
||
|
||
if toPort:
|
||
if toPort != 80 and toPort != 443:
|
||
if ':' not in toDomain:
|
||
toDomain = toDomain + ':' + str(toPort)
|
||
|
||
toDomainUrl = httpPrefix + '://' + toDomain
|
||
if not siteIsActive(toDomainUrl):
|
||
print('Domain is inactive: ' + toDomainUrl)
|
||
return 9
|
||
print('Domain is active: ' + toDomainUrl)
|
||
handleBase = toDomainUrl + '/@'
|
||
if toNickname:
|
||
handle = handleBase + toNickname
|
||
else:
|
||
singleUserInstanceNickname = 'dev'
|
||
handle = handleBase + singleUserInstanceNickname
|
||
|
||
if debug:
|
||
print('DEBUG: handle - ' + handle + ' toPort ' + str(toPort))
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
|
||
domain, projectVersion)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: webfinger for ' + handle + ' failed')
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return 1
|
||
|
||
if wfRequest.get('errors'):
|
||
if debug:
|
||
print('DEBUG: webfinger for ' + handle +
|
||
' failed with errors ' + str(wfRequest['errors']))
|
||
|
||
if not clientToServer:
|
||
postToBox = 'inbox'
|
||
else:
|
||
postToBox = 'outbox'
|
||
|
||
# get the actor inbox/outbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey, toPersonId, sharedInboxUrl, avatarUrl,
|
||
displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix,
|
||
nickname, domain, postToBox)
|
||
|
||
print("inboxUrl: " + str(inboxUrl))
|
||
print("toPersonId: " + str(toPersonId))
|
||
print("sharedInboxUrl: " + str(sharedInboxUrl))
|
||
if inboxUrl:
|
||
if inboxUrl.endswith('/actor/inbox'):
|
||
inboxUrl = sharedInboxUrl
|
||
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: missing inboxUrl')
|
||
return 3
|
||
|
||
if debug:
|
||
print('DEBUG: Sending to endpoint ' + inboxUrl)
|
||
|
||
if not pubKey:
|
||
if debug:
|
||
print('DEBUG: missing pubkey')
|
||
return 4
|
||
if not toPersonId:
|
||
if debug:
|
||
print('DEBUG: missing personId')
|
||
return 5
|
||
# sharedInbox is optional
|
||
|
||
# get the senders private key
|
||
privateKeyPem = getPersonKey(nickname, domain, baseDir, 'private', debug)
|
||
if len(privateKeyPem) == 0:
|
||
if debug:
|
||
print('DEBUG: Private key not found for ' +
|
||
nickname + '@' + domain + ' in ' + baseDir + '/keys/private')
|
||
return 6
|
||
|
||
if toDomain not in inboxUrl:
|
||
if debug:
|
||
print('DEBUG: ' + toDomain + ' is not in ' + inboxUrl)
|
||
return 7
|
||
postPath = inboxUrl.split(toDomain, 1)[1]
|
||
|
||
addFollowersToPublicPost(postJsonObject)
|
||
|
||
if not postJsonObject.get('signature'):
|
||
try:
|
||
signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem)
|
||
postJsonObject = signedPostJsonObject
|
||
except Exception as e:
|
||
print('WARN: failed to JSON-LD sign post, ' + str(e))
|
||
pass
|
||
|
||
# convert json to string so that there are no
|
||
# subsequent conversions after creating message body digest
|
||
postJsonStr = json.dumps(postJsonObject)
|
||
|
||
# construct the http header, including the message body digest
|
||
signatureHeaderJson = \
|
||
createSignedHeader(privateKeyPem, nickname, domain, port,
|
||
toDomain, toPort,
|
||
postPath, httpPrefix, withDigest, postJsonStr)
|
||
|
||
# Keep the number of threads being used small
|
||
while len(sendThreads) > 1000:
|
||
print('WARN: Maximum threads reached - killing send thread')
|
||
sendThreads[0].kill()
|
||
sendThreads.pop(0)
|
||
print('WARN: thread killed')
|
||
|
||
if debug:
|
||
print('DEBUG: starting thread to send post')
|
||
pprint(postJsonObject)
|
||
thr = \
|
||
threadWithTrace(target=threadSendPost,
|
||
args=(session,
|
||
postJsonStr,
|
||
federationList,
|
||
inboxUrl, baseDir,
|
||
signatureHeaderJson.copy(),
|
||
postLog,
|
||
debug), daemon=True)
|
||
sendThreads.append(thr)
|
||
# thr.start()
|
||
return 0
|
||
|
||
|
||
def addToField(activityType: str, postJsonObject: {},
|
||
debug: bool) -> ({}, bool):
|
||
"""The Follow activity doesn't have a 'to' field and so one
|
||
needs to be added so that activity distribution happens in a consistent way
|
||
Returns true if a 'to' field exists or was added
|
||
"""
|
||
if postJsonObject.get('to'):
|
||
return postJsonObject, True
|
||
|
||
if debug:
|
||
pprint(postJsonObject)
|
||
print('DEBUG: no "to" field when sending to named addresses 2')
|
||
|
||
isSameType = False
|
||
toFieldAdded = False
|
||
if postJsonObject.get('object'):
|
||
if isinstance(postJsonObject['object'], str):
|
||
if postJsonObject.get('type'):
|
||
if postJsonObject['type'] == activityType:
|
||
isSameType = True
|
||
if debug:
|
||
print('DEBUG: "to" field assigned to Follow')
|
||
toAddress = postJsonObject['object']
|
||
if '/statuses/' in toAddress:
|
||
toAddress = toAddress.split('/statuses/')[0]
|
||
postJsonObject['to'] = [toAddress]
|
||
toFieldAdded = True
|
||
elif isinstance(postJsonObject['object'], dict):
|
||
if postJsonObject['object'].get('type'):
|
||
if postJsonObject['object']['type'] == activityType:
|
||
isSameType = True
|
||
if isinstance(postJsonObject['object']['object'], str):
|
||
if debug:
|
||
print('DEBUG: "to" field assigned to Follow')
|
||
toAddress = postJsonObject['object']['object']
|
||
if '/statuses/' in toAddress:
|
||
toAddress = toAddress.split('/statuses/')[0]
|
||
postJsonObject['object']['to'] = [toAddress]
|
||
postJsonObject['to'] = \
|
||
[postJsonObject['object']['object']]
|
||
toFieldAdded = True
|
||
|
||
if not isSameType:
|
||
return postJsonObject, True
|
||
if toFieldAdded:
|
||
return postJsonObject, True
|
||
return postJsonObject, False
|
||
|
||
|
||
def sendToNamedAddresses(session, baseDir: str,
|
||
nickname: str, domain: str,
|
||
onionDomain: str, i2pDomain: str, port: int,
|
||
httpPrefix: str, federationList: [],
|
||
sendThreads: [], postLog: [],
|
||
cachedWebfingers: {}, personCache: {},
|
||
postJsonObject: {}, debug: bool,
|
||
projectVersion: str) -> None:
|
||
"""sends a post to the specific named addresses in to/cc
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendToNamedAddresses')
|
||
return
|
||
if not postJsonObject.get('object'):
|
||
return
|
||
if isinstance(postJsonObject['object'], dict):
|
||
isProfileUpdate = False
|
||
# for actor updates there is no 'to' within the object
|
||
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
|
||
if (postJsonObject['type'] == 'Update' and
|
||
(postJsonObject['object']['type'] == 'Person' or
|
||
postJsonObject['object']['type'] == 'Application' or
|
||
postJsonObject['object']['type'] == 'Group' or
|
||
postJsonObject['object']['type'] == 'Service')):
|
||
# use the original object, which has a 'to'
|
||
recipientsObject = postJsonObject
|
||
isProfileUpdate = True
|
||
|
||
if not isProfileUpdate:
|
||
if not postJsonObject['object'].get('to'):
|
||
if debug:
|
||
pprint(postJsonObject)
|
||
print('DEBUG: ' +
|
||
'no "to" field when sending to named addresses')
|
||
if postJsonObject['object'].get('type'):
|
||
if postJsonObject['object']['type'] == 'Follow':
|
||
if isinstance(postJsonObject['object']['object'], str):
|
||
if debug:
|
||
print('DEBUG: "to" field assigned to Follow')
|
||
postJsonObject['object']['to'] = \
|
||
[postJsonObject['object']['object']]
|
||
if not postJsonObject['object'].get('to'):
|
||
return
|
||
recipientsObject = postJsonObject['object']
|
||
else:
|
||
postJsonObject, fieldAdded = \
|
||
addToField('Follow', postJsonObject, debug)
|
||
if not fieldAdded:
|
||
return
|
||
postJsonObject, fieldAdded = addToField('Like', postJsonObject, debug)
|
||
if not fieldAdded:
|
||
return
|
||
recipientsObject = postJsonObject
|
||
|
||
recipients = []
|
||
recipientType = ('to', 'cc')
|
||
for rType in recipientType:
|
||
if not recipientsObject.get(rType):
|
||
continue
|
||
if isinstance(recipientsObject[rType], list):
|
||
if debug:
|
||
pprint(recipientsObject)
|
||
print('recipientsObject: ' + str(recipientsObject[rType]))
|
||
for address in recipientsObject[rType]:
|
||
if not address:
|
||
continue
|
||
if '/' not in address:
|
||
continue
|
||
if address.endswith('#Public'):
|
||
continue
|
||
if address.endswith('/followers'):
|
||
continue
|
||
recipients.append(address)
|
||
elif isinstance(recipientsObject[rType], str):
|
||
address = recipientsObject[rType]
|
||
if address:
|
||
if '/' in address:
|
||
if address.endswith('#Public'):
|
||
continue
|
||
if address.endswith('/followers'):
|
||
continue
|
||
recipients.append(address)
|
||
if not recipients:
|
||
if debug:
|
||
print('DEBUG: no individual recipients')
|
||
return
|
||
if debug:
|
||
print('DEBUG: Sending individually addressed posts: ' +
|
||
str(recipients))
|
||
# this is after the message has arrived at the server
|
||
clientToServer = False
|
||
for address in recipients:
|
||
toNickname = getNicknameFromActor(address)
|
||
if not toNickname:
|
||
continue
|
||
toDomain, toPort = getDomainFromActor(address)
|
||
if not toDomain:
|
||
continue
|
||
if debug:
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
toDomainFull = toDomain
|
||
if toPort:
|
||
if toPort != 80 and toPort != 443:
|
||
if ':' not in toDomain:
|
||
toDomainFull = toDomain + ':' + str(toPort)
|
||
print('DEBUG: Post sending s2s: ' + nickname + '@' + domainFull +
|
||
' to ' + toNickname + '@' + toDomainFull)
|
||
|
||
# if we have an alt onion domain and we are sending to
|
||
# another onion domain then switch the clearnet
|
||
# domain for the onion one
|
||
fromDomain = domain
|
||
fromHttpPrefix = httpPrefix
|
||
if onionDomain:
|
||
if toDomain.endswith('.onion'):
|
||
fromDomain = onionDomain
|
||
fromHttpPrefix = 'http'
|
||
elif i2pDomain:
|
||
if toDomain.endswith('.i2p'):
|
||
fromDomain = i2pDomain
|
||
fromHttpPrefix = 'http'
|
||
cc = []
|
||
sendSignedJson(postJsonObject, session, baseDir,
|
||
nickname, fromDomain, port,
|
||
toNickname, toDomain, toPort,
|
||
cc, fromHttpPrefix, True, clientToServer,
|
||
federationList,
|
||
sendThreads, postLog, cachedWebfingers,
|
||
personCache, debug, projectVersion)
|
||
|
||
|
||
def hasSharedInbox(session, httpPrefix: str, domain: str) -> bool:
|
||
"""Returns true if the given domain has a shared inbox
|
||
"""
|
||
wfRequest = webfingerHandle(session, domain + '@' + domain,
|
||
httpPrefix, {},
|
||
None, __version__)
|
||
if wfRequest:
|
||
if isinstance(wfRequest, dict):
|
||
if not wfRequest.get('errors'):
|
||
return True
|
||
return False
|
||
|
||
|
||
def sendToFollowers(session, baseDir: str,
|
||
nickname: str,
|
||
domain: str,
|
||
onionDomain: str, i2pDomain: str, port: int,
|
||
httpPrefix: str, federationList: [],
|
||
sendThreads: [], postLog: [],
|
||
cachedWebfingers: {}, personCache: {},
|
||
postJsonObject: {}, debug: bool,
|
||
projectVersion: str) -> None:
|
||
"""sends a post to the followers of the given nickname
|
||
"""
|
||
print('sendToFollowers')
|
||
if not session:
|
||
print('WARN: No session for sendToFollowers')
|
||
return
|
||
if not postIsAddressedToFollowers(baseDir, nickname, domain,
|
||
port, httpPrefix,
|
||
postJsonObject):
|
||
if debug:
|
||
print('Post is not addressed to followers')
|
||
return
|
||
print('Post is addressed to followers')
|
||
|
||
grouped = groupFollowersByDomain(baseDir, nickname, domain)
|
||
if not grouped:
|
||
if debug:
|
||
print('Post to followers did not resolve any domains')
|
||
return
|
||
print('Post to followers resolved domains')
|
||
print(str(grouped))
|
||
|
||
# this is after the message has arrived at the server
|
||
clientToServer = False
|
||
|
||
# for each instance
|
||
for followerDomain, followerHandles in grouped.items():
|
||
if debug:
|
||
print('DEBUG: follower handles for ' + followerDomain)
|
||
pprint(followerHandles)
|
||
|
||
# check that the follower's domain is active
|
||
followerDomainUrl = httpPrefix + '://' + followerDomain
|
||
if not siteIsActive(followerDomainUrl):
|
||
print('Domain is inactive: ' + followerDomainUrl)
|
||
continue
|
||
print('Domain is active: ' + followerDomainUrl)
|
||
|
||
withSharedInbox = hasSharedInbox(session, httpPrefix, followerDomain)
|
||
if debug:
|
||
if withSharedInbox:
|
||
print(followerDomain + ' has shared inbox')
|
||
else:
|
||
print(followerDomain + ' does not have a shared inbox')
|
||
|
||
toPort = port
|
||
index = 0
|
||
toDomain = followerHandles[index].split('@')[1]
|
||
if ':' in toDomain:
|
||
toPort = toDomain.split(':')[1]
|
||
toDomain = toDomain.split(':')[0]
|
||
|
||
cc = ''
|
||
|
||
# if we are sending to an onion domain and we
|
||
# have an alt onion domain then use the alt
|
||
fromDomain = domain
|
||
fromHttpPrefix = httpPrefix
|
||
if onionDomain:
|
||
if toDomain.endswith('.onion'):
|
||
fromDomain = onionDomain
|
||
fromHttpPrefix = 'http'
|
||
elif i2pDomain:
|
||
if toDomain.endswith('.i2p'):
|
||
fromDomain = i2pDomain
|
||
fromHttpPrefix = 'http'
|
||
|
||
if withSharedInbox:
|
||
toNickname = followerHandles[index].split('@')[0]
|
||
|
||
# if there are more than one followers on the domain
|
||
# then send the post to the shared inbox
|
||
if len(followerHandles) > 1:
|
||
toNickname = 'inbox'
|
||
|
||
if toNickname != 'inbox' and postJsonObject.get('type'):
|
||
if postJsonObject['type'] == 'Update':
|
||
if postJsonObject.get('object'):
|
||
if isinstance(postJsonObject['object'], dict):
|
||
if postJsonObject['object'].get('type'):
|
||
typ = postJsonObject['object']['type']
|
||
if typ == 'Person' or \
|
||
typ == 'Application' or \
|
||
typ == 'Group' or \
|
||
typ == 'Service':
|
||
print('Sending profile update to ' +
|
||
'shared inbox of ' + toDomain)
|
||
toNickname = 'inbox'
|
||
|
||
if debug:
|
||
print('DEBUG: Sending from ' + nickname + '@' + domain +
|
||
' to ' + toNickname + '@' + toDomain)
|
||
|
||
sendSignedJson(postJsonObject, session, baseDir,
|
||
nickname, fromDomain, port,
|
||
toNickname, toDomain, toPort,
|
||
cc, fromHttpPrefix, True, clientToServer,
|
||
federationList,
|
||
sendThreads, postLog, cachedWebfingers,
|
||
personCache, debug, projectVersion)
|
||
else:
|
||
# send to individual followers without using a shared inbox
|
||
for handle in followerHandles:
|
||
if debug:
|
||
print('DEBUG: Sending to ' + handle)
|
||
toNickname = handle.split('@')[0]
|
||
|
||
if debug:
|
||
if postJsonObject['type'] != 'Update':
|
||
print('DEBUG: Sending from ' +
|
||
nickname + '@' + domain + ' to ' +
|
||
toNickname + '@' + toDomain)
|
||
else:
|
||
print('DEBUG: Sending profile update from ' +
|
||
nickname + '@' + domain + ' to ' +
|
||
toNickname + '@' + toDomain)
|
||
|
||
sendSignedJson(postJsonObject, session, baseDir,
|
||
nickname, fromDomain, port,
|
||
toNickname, toDomain, toPort,
|
||
cc, fromHttpPrefix, True, clientToServer,
|
||
federationList,
|
||
sendThreads, postLog, cachedWebfingers,
|
||
personCache, debug, projectVersion)
|
||
|
||
time.sleep(4)
|
||
|
||
if debug:
|
||
print('DEBUG: End of sendToFollowers')
|
||
|
||
|
||
def sendToFollowersThread(session, baseDir: str,
|
||
nickname: str,
|
||
domain: str,
|
||
onionDomain: str, i2pDomain: str, port: int,
|
||
httpPrefix: str, federationList: [],
|
||
sendThreads: [], postLog: [],
|
||
cachedWebfingers: {}, personCache: {},
|
||
postJsonObject: {}, debug: bool,
|
||
projectVersion: str):
|
||
"""Returns a thread used to send a post to followers
|
||
"""
|
||
sendThread = \
|
||
threadWithTrace(target=sendToFollowers,
|
||
args=(session, baseDir,
|
||
nickname, domain,
|
||
onionDomain, i2pDomain, port,
|
||
httpPrefix, federationList,
|
||
sendThreads, postLog,
|
||
cachedWebfingers, personCache,
|
||
postJsonObject.copy(), debug,
|
||
projectVersion), daemon=True)
|
||
try:
|
||
sendThread.start()
|
||
except SocketError as e:
|
||
print('WARN: socket error while starting ' +
|
||
'thread to send to followers. ' + str(e))
|
||
return None
|
||
except ValueError as e:
|
||
print('WARN: error while starting ' +
|
||
'thread to send to followers. ' + str(e))
|
||
return None
|
||
return sendThread
|
||
|
||
|
||
def createInbox(recentPostsCache: {},
|
||
session, baseDir: str, nickname: str, domain: str, port: int,
|
||
httpPrefix: str, itemsPerPage: int, headerOnly: bool,
|
||
pageNumber=None) -> {}:
|
||
return createBoxIndexed(recentPostsCache,
|
||
session, baseDir, 'inbox',
|
||
nickname, domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'tlbookmarks',
|
||
nickname, domain,
|
||
port, httpPrefix, itemsPerPage, headerOnly,
|
||
True, 0, False, 0, pageNumber)
|
||
|
||
|
||
def createEventsTimeline(recentPostsCache: {},
|
||
session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed(recentPostsCache, session, baseDir, 'tlevents',
|
||
nickname, domain,
|
||
port, httpPrefix, itemsPerPage, headerOnly,
|
||
True, 0, False, 0, pageNumber)
|
||
|
||
|
||
def createDMTimeline(recentPostsCache: {},
|
||
session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed(recentPostsCache,
|
||
session, baseDir, 'dm', nickname,
|
||
domain, port, httpPrefix, itemsPerPage,
|
||
headerOnly, True, 0, False, 0, pageNumber)
|
||
|
||
|
||
def createRepliesTimeline(recentPostsCache: {},
|
||
session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed(recentPostsCache, session, baseDir, 'tlreplies',
|
||
nickname, domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'tlblogs', nickname,
|
||
domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createFeaturesTimeline(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'tlfeatures', nickname,
|
||
domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createMediaTimeline(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'tlmedia', nickname,
|
||
domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createNewsTimeline(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str, itemsPerPage: int,
|
||
headerOnly: bool, newswireVotesThreshold: int,
|
||
positiveVoting: bool, votingTimeMins: int,
|
||
pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'outbox', 'news',
|
||
domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, True,
|
||
newswireVotesThreshold, positiveVoting,
|
||
votingTimeMins, pageNumber)
|
||
|
||
|
||
def createOutbox(session, baseDir: str, nickname: str, domain: str,
|
||
port: int, httpPrefix: str,
|
||
itemsPerPage: int, headerOnly: bool, authorized: bool,
|
||
pageNumber=None) -> {}:
|
||
return createBoxIndexed({}, session, baseDir, 'outbox',
|
||
nickname, domain, port, httpPrefix,
|
||
itemsPerPage, headerOnly, authorized,
|
||
0, False, 0, pageNumber)
|
||
|
||
|
||
def createModeration(baseDir: str, nickname: str, domain: str, port: int,
|
||
httpPrefix: str, itemsPerPage: int, headerOnly: bool,
|
||
pageNumber=None) -> {}:
|
||
boxDir = createPersonDir(nickname, domain, baseDir, 'inbox')
|
||
boxname = 'moderation'
|
||
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domain = domain + ':' + str(port)
|
||
|
||
if not pageNumber:
|
||
pageNumber = 1
|
||
|
||
pageStr = '?page=' + str(pageNumber)
|
||
boxUrl = httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname
|
||
boxHeader = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'first': boxUrl+'?page=true',
|
||
'id': boxUrl,
|
||
'last': boxUrl+'?page=true',
|
||
'totalItems': 0,
|
||
'type': 'OrderedCollection'
|
||
}
|
||
boxItems = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'id': boxUrl+pageStr,
|
||
'orderedItems': [
|
||
],
|
||
'partOf': boxUrl,
|
||
'type': 'OrderedCollectionPage'
|
||
}
|
||
|
||
if isModerator(baseDir, nickname):
|
||
moderationIndexFile = baseDir + '/accounts/moderation.txt'
|
||
if os.path.isfile(moderationIndexFile):
|
||
with open(moderationIndexFile, "r") as f:
|
||
lines = f.readlines()
|
||
boxHeader['totalItems'] = len(lines)
|
||
if headerOnly:
|
||
return boxHeader
|
||
|
||
pageLines = []
|
||
if len(lines) > 0:
|
||
endLineNumber = len(lines) - 1 - int(itemsPerPage * pageNumber)
|
||
if endLineNumber < 0:
|
||
endLineNumber = 0
|
||
startLineNumber = \
|
||
len(lines) - 1 - int(itemsPerPage * (pageNumber - 1))
|
||
if startLineNumber < 0:
|
||
startLineNumber = 0
|
||
lineNumber = startLineNumber
|
||
while lineNumber >= endLineNumber:
|
||
pageLines.append(lines[lineNumber].strip('\n').strip('\r'))
|
||
lineNumber -= 1
|
||
|
||
for postUrl in pageLines:
|
||
postFilename = \
|
||
boxDir + '/' + postUrl.replace('/', '#') + '.json'
|
||
if os.path.isfile(postFilename):
|
||
postJsonObject = loadJson(postFilename)
|
||
if postJsonObject:
|
||
boxItems['orderedItems'].append(postJsonObject)
|
||
|
||
if headerOnly:
|
||
return boxHeader
|
||
return boxItems
|
||
|
||
|
||
def getStatusNumberFromPostFilename(filename) -> int:
|
||
"""Gets the status number from a post filename
|
||
eg. https:##testdomain.com:8085#users#testuser567#
|
||
statuses#1562958506952068.json
|
||
returns 156295850695206
|
||
"""
|
||
if '#statuses#' not in filename:
|
||
return None
|
||
return int(filename.split('#')[-1].replace('.json', ''))
|
||
|
||
|
||
def isDM(postJsonObject: {}) -> bool:
|
||
"""Returns true if the given post is a DM
|
||
"""
|
||
if postJsonObject['type'] != 'Create':
|
||
return False
|
||
if not postJsonObject.get('object'):
|
||
return False
|
||
if not isinstance(postJsonObject['object'], dict):
|
||
return False
|
||
if postJsonObject['object']['type'] != 'Note' and \
|
||
postJsonObject['object']['type'] != 'Patch' and \
|
||
postJsonObject['object']['type'] != 'EncryptedMessage' and \
|
||
postJsonObject['object']['type'] != 'Article':
|
||
return False
|
||
if postJsonObject['object'].get('moderationStatus'):
|
||
return False
|
||
fields = ('to', 'cc')
|
||
for f in fields:
|
||
if not postJsonObject['object'].get(f):
|
||
continue
|
||
for toAddress in postJsonObject['object'][f]:
|
||
if toAddress.endswith('#Public'):
|
||
return False
|
||
if toAddress.endswith('followers'):
|
||
return False
|
||
return True
|
||
|
||
|
||
def isImageMedia(session, baseDir: str, httpPrefix: str,
|
||
nickname: str, domain: str,
|
||
postJsonObject: {}, translate: {},
|
||
YTReplacementDomain: str) -> bool:
|
||
"""Returns true if the given post has attached image media
|
||
"""
|
||
if postJsonObject['type'] == 'Announce':
|
||
postJsonAnnounce = \
|
||
downloadAnnounce(session, baseDir, httpPrefix,
|
||
nickname, domain, postJsonObject,
|
||
__version__, translate,
|
||
YTReplacementDomain)
|
||
if postJsonAnnounce:
|
||
postJsonObject = postJsonAnnounce
|
||
if postJsonObject['type'] != 'Create':
|
||
return False
|
||
if not postJsonObject.get('object'):
|
||
return False
|
||
if not isinstance(postJsonObject['object'], dict):
|
||
return False
|
||
if postJsonObject['object'].get('moderationStatus'):
|
||
return False
|
||
if postJsonObject['object']['type'] != 'Note' and \
|
||
postJsonObject['object']['type'] != 'Event' and \
|
||
postJsonObject['object']['type'] != 'Article':
|
||
return False
|
||
if not postJsonObject['object'].get('attachment'):
|
||
return False
|
||
if not isinstance(postJsonObject['object']['attachment'], list):
|
||
return False
|
||
for attach in postJsonObject['object']['attachment']:
|
||
if attach.get('mediaType') and attach.get('url'):
|
||
if attach['mediaType'].startswith('image/') or \
|
||
attach['mediaType'].startswith('audio/') or \
|
||
attach['mediaType'].startswith('video/'):
|
||
return True
|
||
return False
|
||
|
||
|
||
def isReply(postJsonObject: {}, actor: str) -> bool:
|
||
"""Returns true if the given post is a reply to the given actor
|
||
"""
|
||
if postJsonObject['type'] != 'Create':
|
||
return False
|
||
if not postJsonObject.get('object'):
|
||
return False
|
||
if not isinstance(postJsonObject['object'], dict):
|
||
return False
|
||
if postJsonObject['object'].get('moderationStatus'):
|
||
return False
|
||
if postJsonObject['object']['type'] != 'Note' and \
|
||
postJsonObject['object']['type'] != 'EncryptedMessage' and \
|
||
postJsonObject['object']['type'] != 'Article':
|
||
return False
|
||
if postJsonObject['object'].get('inReplyTo'):
|
||
if isinstance(postJsonObject['object']['inReplyTo'], str):
|
||
if postJsonObject['object']['inReplyTo'].startswith(actor):
|
||
return True
|
||
if not postJsonObject['object'].get('tag'):
|
||
return False
|
||
if not isinstance(postJsonObject['object']['tag'], list):
|
||
return False
|
||
for tag in postJsonObject['object']['tag']:
|
||
if not tag.get('type'):
|
||
continue
|
||
if tag['type'] == 'Mention':
|
||
if not tag.get('href'):
|
||
continue
|
||
if actor in tag['href']:
|
||
return True
|
||
return False
|
||
|
||
|
||
def createBoxIndex(boxDir: str, postsInBoxDict: {}) -> int:
|
||
""" Creates an index for the given box
|
||
"""
|
||
postsCtr = 0
|
||
postsInPersonInbox = os.scandir(boxDir)
|
||
for postFilename in postsInPersonInbox:
|
||
postFilename = postFilename.name
|
||
if not postFilename.endswith('.json'):
|
||
continue
|
||
# extract the status number
|
||
statusNumber = getStatusNumberFromPostFilename(postFilename)
|
||
if statusNumber:
|
||
postsInBoxDict[statusNumber] = os.path.join(boxDir, postFilename)
|
||
postsCtr += 1
|
||
return postsCtr
|
||
|
||
|
||
def createSharedInboxIndex(baseDir: str, sharedBoxDir: str,
|
||
postsInBoxDict: {}, postsCtr: int,
|
||
nickname: str, domain: str) -> int:
|
||
""" Creates an index for the given shared inbox
|
||
"""
|
||
handle = nickname + '@' + domain
|
||
followingFilename = baseDir + '/accounts/' + handle + '/following.txt'
|
||
postsInSharedInbox = os.scandir(sharedBoxDir)
|
||
followingHandles = None
|
||
for postFilename in postsInSharedInbox:
|
||
postFilename = postFilename.name
|
||
if not postFilename.endswith('.json'):
|
||
continue
|
||
statusNumber = getStatusNumberFromPostFilename(postFilename)
|
||
if not statusNumber:
|
||
continue
|
||
|
||
sharedInboxFilename = os.path.join(sharedBoxDir, postFilename)
|
||
# get the actor from the shared post
|
||
postJsonObject = loadJson(sharedInboxFilename, 0)
|
||
if not postJsonObject:
|
||
print('WARN: json load exception createSharedInboxIndex')
|
||
continue
|
||
|
||
actorNickname = getNicknameFromActor(postJsonObject['actor'])
|
||
if not actorNickname:
|
||
continue
|
||
actorDomain, actorPort = getDomainFromActor(postJsonObject['actor'])
|
||
if not actorDomain:
|
||
continue
|
||
|
||
# is the actor followed by this account?
|
||
if not followingHandles:
|
||
with open(followingFilename, 'r') as followingFile:
|
||
followingHandles = followingFile.read()
|
||
if actorNickname + '@' + actorDomain not in followingHandles:
|
||
continue
|
||
|
||
postsInBoxDict[statusNumber] = sharedInboxFilename
|
||
postsCtr += 1
|
||
return postsCtr
|
||
|
||
|
||
def addPostStringToTimeline(postStr: str, boxname: str,
|
||
postsInBox: [], boxActor: str) -> bool:
|
||
""" is this a valid timeline post?
|
||
"""
|
||
# must be a recognized ActivityPub type
|
||
if ('"Note"' in postStr or
|
||
'"EncryptedMessage"' in postStr or
|
||
'"Event"' in postStr or
|
||
'"Article"' in postStr or
|
||
'"Patch"' in postStr or
|
||
'"Announce"' in postStr or
|
||
('"Question"' in postStr and
|
||
('"Create"' in postStr or '"Update"' in postStr))):
|
||
|
||
if boxname == 'dm':
|
||
if '#Public' in postStr or '/followers' in postStr:
|
||
return False
|
||
elif boxname == 'tlreplies':
|
||
if boxActor not in postStr:
|
||
return False
|
||
elif (boxname == 'tlblogs' or
|
||
boxname == 'tlnews' or
|
||
boxname == 'tlfeatures'):
|
||
if '"Create"' not in postStr:
|
||
return False
|
||
if '"Article"' not in postStr:
|
||
return False
|
||
elif boxname == 'tlmedia':
|
||
if '"Create"' in postStr:
|
||
if 'mediaType' not in postStr or 'image/' not in postStr:
|
||
return False
|
||
# add the post to the dictionary
|
||
postsInBox.append(postStr)
|
||
return True
|
||
return False
|
||
|
||
|
||
def addPostToTimeline(filePath: str, boxname: str,
|
||
postsInBox: [], boxActor: str) -> bool:
|
||
""" Reads a post from file and decides whether it is valid
|
||
"""
|
||
with open(filePath, 'r') as postFile:
|
||
postStr = postFile.read()
|
||
|
||
if filePath.endswith('.json'):
|
||
repliesFilename = filePath.replace('.json', '.replies')
|
||
if os.path.isfile(repliesFilename):
|
||
# append a replies identifier, which will later be removed
|
||
postStr += '<hasReplies>'
|
||
|
||
return addPostStringToTimeline(postStr, boxname, postsInBox, boxActor)
|
||
return False
|
||
|
||
|
||
def createBoxIndexed(recentPostsCache: {},
|
||
session, baseDir: str, boxname: str,
|
||
nickname: str, domain: str, port: int, httpPrefix: str,
|
||
itemsPerPage: int, headerOnly: bool, authorized: bool,
|
||
newswireVotesThreshold: int, positiveVoting: bool,
|
||
votingTimeMins: int, pageNumber=None) -> {}:
|
||
"""Constructs the box feed for a person with the given nickname
|
||
"""
|
||
if not authorized or not pageNumber:
|
||
pageNumber = 1
|
||
|
||
if boxname != 'inbox' and boxname != 'dm' and \
|
||
boxname != 'tlreplies' and boxname != 'tlmedia' and \
|
||
boxname != 'tlblogs' and boxname != 'tlnews' and \
|
||
boxname != 'tlfeatures' and \
|
||
boxname != 'outbox' and boxname != 'tlbookmarks' and \
|
||
boxname != 'bookmarks' and \
|
||
boxname != 'tlevents':
|
||
return None
|
||
|
||
# bookmarks and events timelines are like the inbox
|
||
# but have their own separate index
|
||
indexBoxName = boxname
|
||
timelineNickname = nickname
|
||
if boxname == "tlbookmarks":
|
||
boxname = "bookmarks"
|
||
indexBoxName = boxname
|
||
elif boxname == "tlfeatures":
|
||
boxname = "tlblogs"
|
||
indexBoxName = boxname
|
||
timelineNickname = 'news'
|
||
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domain = domain + ':' + str(port)
|
||
|
||
boxActor = httpPrefix + '://' + domain + '/users/' + nickname
|
||
|
||
pageStr = '?page=true'
|
||
if pageNumber:
|
||
if pageNumber < 1:
|
||
pageNumber = 1
|
||
try:
|
||
pageStr = '?page=' + str(pageNumber)
|
||
except BaseException:
|
||
pass
|
||
boxUrl = httpPrefix + '://' + domain + '/users/' + nickname + '/' + boxname
|
||
boxHeader = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'first': boxUrl + '?page=true',
|
||
'id': boxUrl,
|
||
'last': boxUrl + '?page=true',
|
||
'totalItems': 0,
|
||
'type': 'OrderedCollection'
|
||
}
|
||
boxItems = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'id': boxUrl + pageStr,
|
||
'orderedItems': [
|
||
],
|
||
'partOf': boxUrl,
|
||
'type': 'OrderedCollectionPage'
|
||
}
|
||
|
||
postsInBox = []
|
||
|
||
indexFilename = \
|
||
baseDir + '/accounts/' + timelineNickname + '@' + domain + \
|
||
'/' + indexBoxName + '.index'
|
||
postsCtr = 0
|
||
if os.path.isfile(indexFilename):
|
||
maxPostCtr = itemsPerPage * pageNumber
|
||
with open(indexFilename, 'r') as indexFile:
|
||
while postsCtr < maxPostCtr:
|
||
postFilename = indexFile.readline()
|
||
|
||
if not postFilename:
|
||
break
|
||
|
||
# apply votes within this timeline
|
||
if newswireVotesThreshold > 0:
|
||
# note that the presence of an arrival file also indicates
|
||
# that this post is moderated
|
||
arrivalDate = \
|
||
locateNewsArrival(baseDir, domain, postFilename)
|
||
if arrivalDate:
|
||
# how long has elapsed since this post arrived?
|
||
currDate = datetime.datetime.utcnow()
|
||
timeDiffMins = \
|
||
int((currDate - arrivalDate).total_seconds() / 60)
|
||
# has the voting time elapsed?
|
||
if timeDiffMins < votingTimeMins:
|
||
# voting is still happening, so don't add this
|
||
# post to the timeline
|
||
continue
|
||
# if there a votes file for this post?
|
||
votesFilename = \
|
||
locateNewsVotes(baseDir, domain, postFilename)
|
||
if votesFilename:
|
||
# load the votes file and count the votes
|
||
votesJson = loadJson(votesFilename, 0, 2)
|
||
if votesJson:
|
||
if not positiveVoting:
|
||
if votesOnNewswireItem(votesJson) >= \
|
||
newswireVotesThreshold:
|
||
# Too many veto votes.
|
||
# Continue without incrementing
|
||
# the posts counter
|
||
continue
|
||
else:
|
||
if votesOnNewswireItem < \
|
||
newswireVotesThreshold:
|
||
# Not enough votes.
|
||
# Continue without incrementing
|
||
# the posts counter
|
||
continue
|
||
|
||
# Skip through any posts previous to the current page
|
||
if postsCtr < int((pageNumber - 1) * itemsPerPage):
|
||
postsCtr += 1
|
||
continue
|
||
|
||
# if this is a full path then remove the directories
|
||
if '/' in postFilename:
|
||
postFilename = postFilename.split('/')[-1]
|
||
|
||
# filename of the post without any extension or path
|
||
# This should also correspond to any index entry in
|
||
# the posts cache
|
||
postUrl = \
|
||
postFilename.replace('\n', '').replace('\r', '')
|
||
postUrl = postUrl.replace('.json', '').strip()
|
||
|
||
# is the post cached in memory?
|
||
if recentPostsCache.get('index'):
|
||
if postUrl in recentPostsCache['index']:
|
||
if recentPostsCache['json'].get(postUrl):
|
||
url = recentPostsCache['json'][postUrl]
|
||
addPostStringToTimeline(url,
|
||
boxname, postsInBox,
|
||
boxActor)
|
||
postsCtr += 1
|
||
continue
|
||
|
||
# read the post from file
|
||
fullPostFilename = \
|
||
locatePost(baseDir, nickname,
|
||
domain, postUrl, False)
|
||
if fullPostFilename:
|
||
addPostToTimeline(fullPostFilename, boxname,
|
||
postsInBox, boxActor)
|
||
else:
|
||
# if this is the features timeline
|
||
if timelineNickname != nickname:
|
||
fullPostFilename = \
|
||
locatePost(baseDir, timelineNickname,
|
||
domain, postUrl, False)
|
||
if fullPostFilename:
|
||
addPostToTimeline(fullPostFilename, boxname,
|
||
postsInBox, boxActor)
|
||
else:
|
||
print('WARN: unable to locate post ' + postUrl)
|
||
else:
|
||
print('WARN: unable to locate post ' + postUrl)
|
||
|
||
postsCtr += 1
|
||
|
||
# Generate first and last entries within header
|
||
if postsCtr > 0:
|
||
lastPage = int(postsCtr / itemsPerPage)
|
||
if lastPage < 1:
|
||
lastPage = 1
|
||
boxHeader['last'] = \
|
||
httpPrefix + '://' + domain + '/users/' + \
|
||
nickname + '/' + boxname + '?page=' + str(lastPage)
|
||
|
||
if headerOnly:
|
||
boxHeader['totalItems'] = len(postsInBox)
|
||
prevPageStr = 'true'
|
||
if pageNumber > 1:
|
||
prevPageStr = str(pageNumber - 1)
|
||
boxHeader['prev'] = \
|
||
httpPrefix + '://' + domain + '/users/' + \
|
||
nickname + '/' + boxname + '?page=' + prevPageStr
|
||
|
||
nextPageStr = str(pageNumber + 1)
|
||
boxHeader['next'] = \
|
||
httpPrefix + '://' + domain + '/users/' + \
|
||
nickname + '/' + boxname + '?page=' + nextPageStr
|
||
return boxHeader
|
||
|
||
for postStr in postsInBox:
|
||
# Check if the post has replies
|
||
hasReplies = False
|
||
if postStr.endswith('<hasReplies>'):
|
||
hasReplies = True
|
||
# remove the replies identifier
|
||
postStr = postStr.replace('<hasReplies>', '')
|
||
|
||
p = None
|
||
try:
|
||
p = json.loads(postStr)
|
||
except BaseException:
|
||
continue
|
||
|
||
# Does this post have replies?
|
||
# This will be used to indicate that replies exist within the html
|
||
# created by individualPostAsHtml
|
||
p['hasReplies'] = hasReplies
|
||
|
||
# Don't show likes, replies or shares (announces) to
|
||
# unauthorized viewers
|
||
if not authorized:
|
||
if p.get('object'):
|
||
if isinstance(p['object'], dict):
|
||
if p['object'].get('likes'):
|
||
p['likes'] = {'items': []}
|
||
if p['object'].get('replies'):
|
||
p['replies'] = {}
|
||
if p['object'].get('shares'):
|
||
p['shares'] = {}
|
||
if p['object'].get('bookmarks'):
|
||
p['bookmarks'] = {}
|
||
|
||
boxItems['orderedItems'].append(p)
|
||
|
||
return boxItems
|
||
|
||
|
||
def expireCache(baseDir: str, personCache: {},
|
||
httpPrefix: str, archiveDir: str,
|
||
recentPostsCache: {},
|
||
maxPostsInBox=32000):
|
||
"""Thread used to expire actors from the cache and archive old posts
|
||
"""
|
||
while True:
|
||
# once per day
|
||
time.sleep(60 * 60 * 24)
|
||
expirePersonCache(baseDir, personCache)
|
||
archivePosts(baseDir, httpPrefix, archiveDir, recentPostsCache,
|
||
maxPostsInBox)
|
||
|
||
|
||
def archivePosts(baseDir: str, httpPrefix: str, archiveDir: str,
|
||
recentPostsCache: {},
|
||
maxPostsInBox=32000) -> None:
|
||
"""Archives posts for all accounts
|
||
"""
|
||
if maxPostsInBox == 0:
|
||
return
|
||
|
||
if archiveDir:
|
||
if not os.path.isdir(archiveDir):
|
||
os.mkdir(archiveDir)
|
||
|
||
if archiveDir:
|
||
if not os.path.isdir(archiveDir + '/accounts'):
|
||
os.mkdir(archiveDir + '/accounts')
|
||
|
||
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
|
||
for handle in dirs:
|
||
if '@' in handle:
|
||
nickname = handle.split('@')[0]
|
||
domain = handle.split('@')[1]
|
||
archiveSubdir = None
|
||
if archiveDir:
|
||
if not os.path.isdir(archiveDir + '/accounts/' + handle):
|
||
os.mkdir(archiveDir + '/accounts/' + handle)
|
||
if not os.path.isdir(archiveDir + '/accounts/' +
|
||
handle + '/inbox'):
|
||
os.mkdir(archiveDir + '/accounts/' +
|
||
handle + '/inbox')
|
||
if not os.path.isdir(archiveDir + '/accounts/' +
|
||
handle + '/outbox'):
|
||
os.mkdir(archiveDir + '/accounts/' +
|
||
handle + '/outbox')
|
||
archiveSubdir = archiveDir + '/accounts/' + \
|
||
handle + '/inbox'
|
||
archivePostsForPerson(httpPrefix, nickname, domain, baseDir,
|
||
'inbox', archiveSubdir,
|
||
recentPostsCache, maxPostsInBox)
|
||
if archiveDir:
|
||
archiveSubdir = archiveDir + '/accounts/' + \
|
||
handle + '/outbox'
|
||
archivePostsForPerson(httpPrefix, nickname, domain, baseDir,
|
||
'outbox', archiveSubdir,
|
||
recentPostsCache, maxPostsInBox)
|
||
|
||
|
||
def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,
|
||
baseDir: str,
|
||
boxname: str, archiveDir: str,
|
||
recentPostsCache: {},
|
||
maxPostsInBox=32000) -> None:
|
||
"""Retain a maximum number of posts within the given box
|
||
Move any others to an archive directory
|
||
"""
|
||
if boxname != 'inbox' and boxname != 'outbox':
|
||
return
|
||
if archiveDir:
|
||
if not os.path.isdir(archiveDir):
|
||
os.mkdir(archiveDir)
|
||
boxDir = createPersonDir(nickname, domain, baseDir, boxname)
|
||
postsInBox = os.scandir(boxDir)
|
||
noOfPosts = 0
|
||
for f in postsInBox:
|
||
noOfPosts += 1
|
||
if noOfPosts <= maxPostsInBox:
|
||
print('Checked ' + str(noOfPosts) + ' ' + boxname +
|
||
' posts for ' + nickname + '@' + domain)
|
||
return
|
||
|
||
# remove entries from the index
|
||
handle = nickname + '@' + domain
|
||
indexFilename = baseDir + '/accounts/' + handle + '/' + boxname + '.index'
|
||
if os.path.isfile(indexFilename):
|
||
indexCtr = 0
|
||
# get the existing index entries as a string
|
||
newIndex = ''
|
||
with open(indexFilename, 'r') as indexFile:
|
||
for postId in indexFile:
|
||
newIndex += postId
|
||
indexCtr += 1
|
||
if indexCtr >= maxPostsInBox:
|
||
break
|
||
# save the new index file
|
||
if len(newIndex) > 0:
|
||
indexFile = open(indexFilename, 'w+')
|
||
if indexFile:
|
||
indexFile.write(newIndex)
|
||
indexFile.close()
|
||
|
||
postsInBoxDict = {}
|
||
postsCtr = 0
|
||
postsInBox = os.scandir(boxDir)
|
||
for postFilename in postsInBox:
|
||
postFilename = postFilename.name
|
||
if not postFilename.endswith('.json'):
|
||
continue
|
||
# Time of file creation
|
||
fullFilename = os.path.join(boxDir, postFilename)
|
||
if os.path.isfile(fullFilename):
|
||
content = open(fullFilename).read()
|
||
if '"published":' in content:
|
||
publishedStr = content.split('"published":')[1]
|
||
if '"' in publishedStr:
|
||
publishedStr = publishedStr.split('"')[1]
|
||
if publishedStr.endswith('Z'):
|
||
postsInBoxDict[publishedStr] = postFilename
|
||
postsCtr += 1
|
||
|
||
noOfPosts = postsCtr
|
||
if noOfPosts <= maxPostsInBox:
|
||
print('Checked ' + str(noOfPosts) + ' ' + boxname +
|
||
' posts for ' + nickname + '@' + domain)
|
||
return
|
||
|
||
# sort the list in ascending order of date
|
||
postsInBoxSorted = \
|
||
OrderedDict(sorted(postsInBoxDict.items(), reverse=False))
|
||
|
||
# directory containing cached html posts
|
||
postCacheDir = boxDir.replace('/' + boxname, '/postcache')
|
||
|
||
removeCtr = 0
|
||
for publishedStr, postFilename in postsInBoxSorted.items():
|
||
filePath = os.path.join(boxDir, postFilename)
|
||
if not os.path.isfile(filePath):
|
||
continue
|
||
if archiveDir:
|
||
archivePath = os.path.join(archiveDir, postFilename)
|
||
os.rename(filePath, archivePath)
|
||
|
||
extensions = ('replies', 'votes', 'arrived', 'muted')
|
||
for ext in extensions:
|
||
extPath = filePath.replace('.json', '.' + ext)
|
||
if os.path.isfile(extPath):
|
||
os.rename(extPath,
|
||
archivePath.replace('.json', '.' + ext))
|
||
else:
|
||
extPath = filePath.replace('.json',
|
||
'.json.' + ext)
|
||
if os.path.isfile(extPath):
|
||
os.rename(extPath,
|
||
archivePath.replace('.json', '.json.' + ext))
|
||
else:
|
||
deletePost(baseDir, httpPrefix, nickname, domain,
|
||
filePath, False, recentPostsCache)
|
||
|
||
# remove cached html posts
|
||
postCacheFilename = \
|
||
os.path.join(postCacheDir, postFilename).replace('.json', '.html')
|
||
if os.path.isfile(postCacheFilename):
|
||
os.remove(postCacheFilename)
|
||
|
||
noOfPosts -= 1
|
||
removeCtr += 1
|
||
if noOfPosts <= maxPostsInBox:
|
||
break
|
||
if archiveDir:
|
||
print('Archived ' + str(removeCtr) + ' ' + boxname +
|
||
' posts for ' + nickname + '@' + domain)
|
||
else:
|
||
print('Removed ' + str(removeCtr) + ' ' + boxname +
|
||
' posts for ' + nickname + '@' + domain)
|
||
print(nickname + '@' + domain + ' has ' + str(noOfPosts) +
|
||
' in ' + boxname)
|
||
|
||
|
||
def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
|
||
raw: bool, simple: bool, proxyType: str,
|
||
port: int, httpPrefix: str,
|
||
debug: bool, projectVersion: str) -> None:
|
||
""" This is really just for test purposes
|
||
"""
|
||
print('Starting new session for getting public posts')
|
||
session = createSession(proxyType)
|
||
if not session:
|
||
return
|
||
personCache = {}
|
||
cachedWebfingers = {}
|
||
federationList = []
|
||
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
handle = httpPrefix + "://" + domainFull + "/@" + nickname
|
||
wfRequest = \
|
||
webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
|
||
domain, projectVersion)
|
||
if not wfRequest:
|
||
sys.exit()
|
||
if not isinstance(wfRequest, dict):
|
||
print('Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
sys.exit()
|
||
|
||
(personUrl, pubKeyId, pubKey,
|
||
personId, shaedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix,
|
||
nickname, domain, 'outbox')
|
||
maxMentions = 10
|
||
maxEmoji = 10
|
||
maxAttachments = 5
|
||
getPosts(session, personUrl, 30, maxMentions, maxEmoji,
|
||
maxAttachments, federationList,
|
||
personCache, raw, simple, debug,
|
||
projectVersion, httpPrefix, domain)
|
||
|
||
|
||
def getPublicPostDomains(session, baseDir: str, nickname: str, domain: str,
|
||
proxyType: str, port: int, httpPrefix: str,
|
||
debug: bool, projectVersion: str,
|
||
domainList=[]) -> []:
|
||
""" Returns a list of domains referenced within public posts
|
||
"""
|
||
if not session:
|
||
session = createSession(proxyType)
|
||
if not session:
|
||
return domainList
|
||
personCache = {}
|
||
cachedWebfingers = {}
|
||
federationList = []
|
||
|
||
domainFull = domain
|
||
if port:
|
||
if port != 80 and port != 443:
|
||
if ':' not in domain:
|
||
domainFull = domain + ':' + str(port)
|
||
handle = httpPrefix + "://" + domainFull + "/@" + nickname
|
||
wfRequest = \
|
||
webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
|
||
domain, projectVersion)
|
||
if not wfRequest:
|
||
return domainList
|
||
if not isinstance(wfRequest, dict):
|
||
print('Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return domainList
|
||
|
||
(personUrl, pubKeyId, pubKey,
|
||
personId, sharedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix,
|
||
nickname, domain, 'outbox')
|
||
maxMentions = 99
|
||
maxEmoji = 99
|
||
maxAttachments = 5
|
||
postDomains = \
|
||
getPostDomains(session, personUrl, 64, maxMentions, maxEmoji,
|
||
maxAttachments, federationList,
|
||
personCache, debug,
|
||
projectVersion, httpPrefix, domain, domainList)
|
||
postDomains.sort()
|
||
return postDomains
|
||
|
||
|
||
def getPublicPostDomainsBlocked(session, baseDir: str,
|
||
nickname: str, domain: str,
|
||
proxyType: str, port: int, httpPrefix: str,
|
||
debug: bool, projectVersion: str,
|
||
domainList=[]) -> []:
|
||
""" Returns a list of domains referenced within public posts which
|
||
are globally blocked on this instance
|
||
"""
|
||
postDomains = \
|
||
getPublicPostDomains(session, baseDir, nickname, domain,
|
||
proxyType, port, httpPrefix,
|
||
debug, projectVersion,
|
||
domainList)
|
||
if not postDomains:
|
||
return []
|
||
|
||
blockingFilename = baseDir + '/accounts/blocking.txt'
|
||
if not os.path.isfile(blockingFilename):
|
||
return []
|
||
|
||
# read the blocked domains as a single string
|
||
blockedStr = ''
|
||
with open(blockingFilename, 'r') as fp:
|
||
blockedStr = fp.read()
|
||
|
||
blockedDomains = []
|
||
for domainName in postDomains:
|
||
if '@' not in domainName:
|
||
continue
|
||
# get the domain after the @
|
||
domainName = domainName.split('@')[1].strip()
|
||
if isEvil(domainName):
|
||
blockedDomains.append(domainName)
|
||
continue
|
||
if domainName in blockedStr:
|
||
blockedDomains.append(domainName)
|
||
|
||
return blockedDomains
|
||
|
||
|
||
def getNonMutualsOfPerson(baseDir: str,
|
||
nickname: str, domain: str) -> []:
|
||
"""Returns the followers who are not mutuals of a person
|
||
i.e. accounts which follow you but you don't follow them
|
||
"""
|
||
followers = \
|
||
getFollowersList(baseDir, nickname, domain, 'followers.txt')
|
||
following = \
|
||
getFollowersList(baseDir, nickname, domain, 'following.txt')
|
||
nonMutuals = []
|
||
for handle in followers:
|
||
if handle not in following:
|
||
nonMutuals.append(handle)
|
||
return nonMutuals
|
||
|
||
|
||
def checkDomains(session, baseDir: str,
|
||
nickname: str, domain: str,
|
||
proxyType: str, port: int, httpPrefix: str,
|
||
debug: bool, projectVersion: str,
|
||
maxBlockedDomains: int, singleCheck: bool):
|
||
"""Checks follower accounts for references to globally blocked domains
|
||
"""
|
||
nonMutuals = getNonMutualsOfPerson(baseDir, nickname, domain)
|
||
if not nonMutuals:
|
||
print('No non-mutual followers were found')
|
||
return
|
||
followerWarningFilename = baseDir + '/accounts/followerWarnings.txt'
|
||
updateFollowerWarnings = False
|
||
followerWarningStr = ''
|
||
if os.path.isfile(followerWarningFilename):
|
||
with open(followerWarningFilename, 'r') as fp:
|
||
followerWarningStr = fp.read()
|
||
|
||
if singleCheck:
|
||
# checks a single random non-mutual
|
||
index = random.randrange(0, len(nonMutuals))
|
||
handle = nonMutuals[index]
|
||
if '@' in handle:
|
||
nonMutualNickname = handle.split('@')[0]
|
||
nonMutualDomain = handle.split('@')[1].strip()
|
||
blockedDomains = \
|
||
getPublicPostDomainsBlocked(session, baseDir,
|
||
nonMutualNickname,
|
||
nonMutualDomain,
|
||
proxyType, port, httpPrefix,
|
||
debug, projectVersion, [])
|
||
if blockedDomains:
|
||
if len(blockedDomains) > maxBlockedDomains:
|
||
followerWarningStr += handle + '\n'
|
||
updateFollowerWarnings = True
|
||
else:
|
||
# checks all non-mutuals
|
||
for handle in nonMutuals:
|
||
if '@' not in handle:
|
||
continue
|
||
if handle in followerWarningStr:
|
||
continue
|
||
nonMutualNickname = handle.split('@')[0]
|
||
nonMutualDomain = handle.split('@')[1].strip()
|
||
blockedDomains = \
|
||
getPublicPostDomainsBlocked(session, baseDir,
|
||
nonMutualNickname,
|
||
nonMutualDomain,
|
||
proxyType, port, httpPrefix,
|
||
debug, projectVersion, [])
|
||
if blockedDomains:
|
||
print(handle)
|
||
for d in blockedDomains:
|
||
print(' ' + d)
|
||
if len(blockedDomains) > maxBlockedDomains:
|
||
followerWarningStr += handle + '\n'
|
||
updateFollowerWarnings = True
|
||
|
||
if updateFollowerWarnings and followerWarningStr:
|
||
with open(followerWarningFilename, 'w+') as fp:
|
||
fp.write(followerWarningStr)
|
||
if not singleCheck:
|
||
print(followerWarningStr)
|
||
|
||
|
||
def populateRepliesJson(baseDir: str, nickname: str, domain: str,
|
||
postRepliesFilename: str, authorized: bool,
|
||
repliesJson: {}) -> None:
|
||
pubStr = 'https://www.w3.org/ns/activitystreams#Public'
|
||
# populate the items list with replies
|
||
repliesBoxes = ('outbox', 'inbox')
|
||
with open(postRepliesFilename, 'r') as repliesFile:
|
||
for messageId in repliesFile:
|
||
replyFound = False
|
||
# examine inbox and outbox
|
||
for boxname in repliesBoxes:
|
||
messageId2 = messageId.replace('\n', '').replace('\r', '')
|
||
searchFilename = \
|
||
baseDir + \
|
||
'/accounts/' + nickname + '@' + \
|
||
domain+'/' + \
|
||
boxname+'/' + \
|
||
messageId2.replace('/', '#') + '.json'
|
||
if os.path.isfile(searchFilename):
|
||
if authorized or \
|
||
pubStr in open(searchFilename).read():
|
||
postJsonObject = loadJson(searchFilename)
|
||
if postJsonObject:
|
||
if postJsonObject['object'].get('cc'):
|
||
pjo = postJsonObject
|
||
if (authorized or
|
||
(pubStr in pjo['object']['to'] or
|
||
pubStr in pjo['object']['cc'])):
|
||
repliesJson['orderedItems'].append(pjo)
|
||
replyFound = True
|
||
else:
|
||
if authorized or \
|
||
pubStr in postJsonObject['object']['to']:
|
||
pjo = postJsonObject
|
||
repliesJson['orderedItems'].append(pjo)
|
||
replyFound = True
|
||
break
|
||
# if not in either inbox or outbox then examine the shared inbox
|
||
if not replyFound:
|
||
messageId2 = messageId.replace('\n', '').replace('\r', '')
|
||
searchFilename = \
|
||
baseDir + \
|
||
'/accounts/inbox@' + \
|
||
domain+'/inbox/' + \
|
||
messageId2.replace('/', '#') + '.json'
|
||
if os.path.isfile(searchFilename):
|
||
if authorized or \
|
||
pubStr in open(searchFilename).read():
|
||
# get the json of the reply and append it to
|
||
# the collection
|
||
postJsonObject = loadJson(searchFilename)
|
||
if postJsonObject:
|
||
if postJsonObject['object'].get('cc'):
|
||
pjo = postJsonObject
|
||
if (authorized or
|
||
(pubStr in pjo['object']['to'] or
|
||
pubStr in pjo['object']['cc'])):
|
||
pjo = postJsonObject
|
||
repliesJson['orderedItems'].append(pjo)
|
||
else:
|
||
if authorized or \
|
||
pubStr in postJsonObject['object']['to']:
|
||
pjo = postJsonObject
|
||
repliesJson['orderedItems'].append(pjo)
|
||
|
||
|
||
def rejectAnnounce(announceFilename: str):
|
||
"""Marks an announce as rejected
|
||
"""
|
||
if not os.path.isfile(announceFilename + '.reject'):
|
||
rejectAnnounceFile = open(announceFilename + '.reject', "w+")
|
||
if rejectAnnounceFile:
|
||
rejectAnnounceFile.write('\n')
|
||
rejectAnnounceFile.close()
|
||
|
||
|
||
def downloadAnnounce(session, baseDir: str, httpPrefix: str,
|
||
nickname: str, domain: str,
|
||
postJsonObject: {}, projectVersion: str,
|
||
translate: {}, YTReplacementDomain: str) -> {}:
|
||
"""Download the post referenced by an announce
|
||
"""
|
||
if not postJsonObject.get('object'):
|
||
return None
|
||
if not isinstance(postJsonObject['object'], str):
|
||
return None
|
||
|
||
# get the announced post
|
||
announceCacheDir = baseDir + '/cache/announce/' + nickname
|
||
if not os.path.isdir(announceCacheDir):
|
||
os.mkdir(announceCacheDir)
|
||
announceFilename = \
|
||
announceCacheDir + '/' + \
|
||
postJsonObject['object'].replace('/', '#') + '.json'
|
||
|
||
if os.path.isfile(announceFilename + '.reject'):
|
||
return None
|
||
|
||
if os.path.isfile(announceFilename):
|
||
print('Reading cached Announce content for ' +
|
||
postJsonObject['object'])
|
||
postJsonObject = loadJson(announceFilename)
|
||
if postJsonObject:
|
||
return postJsonObject
|
||
else:
|
||
profileStr = 'https://www.w3.org/ns/activitystreams'
|
||
asHeader = {
|
||
'Accept': 'application/activity+json; profile="' + profileStr + '"'
|
||
}
|
||
if '/channel/' in postJsonObject['actor'] or \
|
||
'/accounts/' in postJsonObject['actor']:
|
||
asHeader = {
|
||
'Accept': 'application/ld+json; profile="' + profileStr + '"'
|
||
}
|
||
actorNickname = getNicknameFromActor(postJsonObject['actor'])
|
||
actorDomain, actorPort = getDomainFromActor(postJsonObject['actor'])
|
||
if not actorDomain:
|
||
print('Announce actor does not contain a ' +
|
||
'valid domain or port number: ' +
|
||
str(postJsonObject['actor']))
|
||
return None
|
||
if isBlocked(baseDir, nickname, domain, actorNickname, actorDomain):
|
||
print('Announce download blocked actor: ' +
|
||
actorNickname + '@' + actorDomain)
|
||
return None
|
||
objectNickname = getNicknameFromActor(postJsonObject['object'])
|
||
objectDomain, objectPort = getDomainFromActor(postJsonObject['object'])
|
||
if not objectDomain:
|
||
print('Announce object does not contain a ' +
|
||
'valid domain or port number: ' +
|
||
str(postJsonObject['object']))
|
||
return None
|
||
if isBlocked(baseDir, nickname, domain, objectNickname, objectDomain):
|
||
if objectNickname and objectDomain:
|
||
print('Announce download blocked object: ' +
|
||
objectNickname + '@' + objectDomain)
|
||
else:
|
||
print('Announce download blocked object: ' +
|
||
str(postJsonObject['object']))
|
||
return None
|
||
print('Downloading Announce content for ' + postJsonObject['object'])
|
||
announcedJson = \
|
||
getJson(session, postJsonObject['object'], asHeader,
|
||
None, projectVersion, httpPrefix, domain)
|
||
|
||
if not announcedJson:
|
||
return None
|
||
|
||
if not isinstance(announcedJson, dict):
|
||
print('WARN: announce json is not a dict - ' +
|
||
postJsonObject['object'])
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
if not announcedJson.get('id'):
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
if '/statuses/' not in announcedJson['id']:
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
if '/users/' not in announcedJson['id'] and \
|
||
'/accounts/' not in announcedJson['id'] and \
|
||
'/channel/' not in announcedJson['id'] and \
|
||
'/profile/' not in announcedJson['id']:
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
if not announcedJson.get('type'):
|
||
rejectAnnounce(announceFilename)
|
||
# pprint(announcedJson)
|
||
return None
|
||
if announcedJson['type'] != 'Note' and \
|
||
announcedJson['type'] != 'Article':
|
||
rejectAnnounce(announceFilename)
|
||
# pprint(announcedJson)
|
||
return None
|
||
if not announcedJson.get('content'):
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
if isFiltered(baseDir, nickname, domain, announcedJson['content']):
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
# remove any long words
|
||
announcedJson['content'] = \
|
||
removeLongWords(announcedJson['content'], 40, [])
|
||
|
||
# remove text formatting, such as bold/italics
|
||
announcedJson['content'] = \
|
||
removeTextFormatting(announcedJson['content'])
|
||
|
||
# wrap in create to be consistent with other posts
|
||
announcedJson = \
|
||
outboxMessageCreateWrap(httpPrefix,
|
||
actorNickname, actorDomain, actorPort,
|
||
announcedJson)
|
||
if announcedJson['type'] != 'Create':
|
||
rejectAnnounce(announceFilename)
|
||
# pprint(announcedJson)
|
||
return None
|
||
|
||
# labelAccusatoryPost(postJsonObject, translate)
|
||
# set the id to the original status
|
||
announcedJson['id'] = postJsonObject['object']
|
||
announcedJson['object']['id'] = postJsonObject['object']
|
||
# check that the repeat isn't for a blocked account
|
||
attributedNickname = \
|
||
getNicknameFromActor(announcedJson['object']['id'])
|
||
attributedDomain, attributedPort = \
|
||
getDomainFromActor(announcedJson['object']['id'])
|
||
if attributedNickname and attributedDomain:
|
||
if attributedPort:
|
||
if attributedPort != 80 and attributedPort != 443:
|
||
attributedDomain = \
|
||
attributedDomain + ':' + str(attributedPort)
|
||
if isBlocked(baseDir, nickname, domain,
|
||
attributedNickname, attributedDomain):
|
||
rejectAnnounce(announceFilename)
|
||
return None
|
||
postJsonObject = announcedJson
|
||
replaceYouTube(postJsonObject, YTReplacementDomain)
|
||
if saveJson(postJsonObject, announceFilename):
|
||
return postJsonObject
|
||
return None
|
||
|
||
|
||
def isMuted(baseDir: str, nickname: str, domain: str, postId: str) -> bool:
|
||
"""Returns true if the given post is muted
|
||
"""
|
||
postFilename = locatePost(baseDir, nickname, domain, postId)
|
||
if not postFilename:
|
||
return False
|
||
if os.path.isfile(postFilename + '.muted'):
|
||
return True
|
||
return False
|
||
|
||
|
||
def mutePost(baseDir: str, nickname: str, domain: str, postId: str,
|
||
recentPostsCache: {}) -> None:
|
||
""" Mutes the given post
|
||
"""
|
||
postFilename = locatePost(baseDir, nickname, domain, postId)
|
||
if not postFilename:
|
||
return
|
||
postJsonObject = loadJson(postFilename)
|
||
if not postJsonObject:
|
||
return
|
||
|
||
# remove cached post so that the muted version gets recreated
|
||
# without its content text and/or image
|
||
cachedPostFilename = \
|
||
getCachedPostFilename(baseDir, nickname, domain, postJsonObject)
|
||
if cachedPostFilename:
|
||
if os.path.isfile(cachedPostFilename):
|
||
os.remove(cachedPostFilename)
|
||
|
||
muteFile = open(postFilename + '.muted', 'w+')
|
||
if muteFile:
|
||
muteFile.write('\n')
|
||
muteFile.close()
|
||
print('MUTE: ' + postFilename + '.muted file added')
|
||
|
||
# if the post is in the recent posts cache then mark it as muted
|
||
if recentPostsCache.get('index'):
|
||
postId = \
|
||
removeIdEnding(postJsonObject['id']).replace('/', '#')
|
||
if postId in recentPostsCache['index']:
|
||
print('MUTE: ' + postId + ' is in recent posts cache')
|
||
if recentPostsCache['json'].get(postId):
|
||
postJsonObject['muted'] = True
|
||
recentPostsCache['json'][postId] = json.dumps(postJsonObject)
|
||
if recentPostsCache.get('html'):
|
||
if recentPostsCache['html'].get(postId):
|
||
del recentPostsCache['html'][postId]
|
||
print('MUTE: ' + postId +
|
||
' marked as muted in recent posts memory cache')
|
||
|
||
|
||
def unmutePost(baseDir: str, nickname: str, domain: str, postId: str,
|
||
recentPostsCache: {}) -> None:
|
||
""" Unmutes the given post
|
||
"""
|
||
postFilename = locatePost(baseDir, nickname, domain, postId)
|
||
if not postFilename:
|
||
return
|
||
postJsonObject = loadJson(postFilename)
|
||
if not postJsonObject:
|
||
return
|
||
|
||
muteFilename = postFilename + '.muted'
|
||
if os.path.isfile(muteFilename):
|
||
os.remove(muteFilename)
|
||
print('UNMUTE: ' + muteFilename + ' file removed')
|
||
|
||
# remove cached post so that the muted version gets recreated
|
||
# with its content text and/or image
|
||
cachedPostFilename = \
|
||
getCachedPostFilename(baseDir, nickname, domain, postJsonObject)
|
||
if cachedPostFilename:
|
||
if os.path.isfile(cachedPostFilename):
|
||
os.remove(cachedPostFilename)
|
||
|
||
# if the post is in the recent posts cache then mark it as unmuted
|
||
if recentPostsCache.get('index'):
|
||
postId = \
|
||
removeIdEnding(postJsonObject['id']).replace('/', '#')
|
||
if postId in recentPostsCache['index']:
|
||
print('UNMUTE: ' + postId + ' is in recent posts cache')
|
||
if recentPostsCache['json'].get(postId):
|
||
postJsonObject['muted'] = False
|
||
recentPostsCache['json'][postId] = json.dumps(postJsonObject)
|
||
if recentPostsCache.get('html'):
|
||
if recentPostsCache['html'].get(postId):
|
||
del recentPostsCache['html'][postId]
|
||
print('UNMUTE: ' + postId +
|
||
' marked as unmuted in recent posts cache')
|
||
|
||
|
||
def sendBlockViaServer(baseDir: str, session,
|
||
fromNickname: str, password: str,
|
||
fromDomain: str, fromPort: int,
|
||
httpPrefix: str, blockedUrl: str,
|
||
cachedWebfingers: {}, personCache: {},
|
||
debug: bool, projectVersion: str) -> {}:
|
||
"""Creates a block via c2s
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendBlockViaServer')
|
||
return 6
|
||
|
||
fromDomainFull = fromDomain
|
||
if fromPort:
|
||
if fromPort != 80 and fromPort != 443:
|
||
if ':' not in fromDomain:
|
||
fromDomainFull = fromDomain + ':' + str(fromPort)
|
||
|
||
toUrl = 'https://www.w3.org/ns/activitystreams#Public'
|
||
ccUrl = httpPrefix + '://' + fromDomainFull + '/users/' + \
|
||
fromNickname + '/followers'
|
||
|
||
blockActor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
|
||
newBlockJson = {
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
'type': 'Block',
|
||
'actor': blockActor,
|
||
'object': blockedUrl,
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
}
|
||
|
||
handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = webfingerHandle(session, handle, httpPrefix,
|
||
cachedWebfingers,
|
||
fromDomain, projectVersion)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: announce webfinger failed for ' + handle)
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return 1
|
||
|
||
postToBox = 'outbox'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
fromPersonId, sharedInbox, avatarUrl,
|
||
displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache,
|
||
projectVersion, httpPrefix, fromNickname,
|
||
fromDomain, postToBox)
|
||
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: No ' + postToBox + ' was found for ' + handle)
|
||
return 3
|
||
if not fromPersonId:
|
||
if debug:
|
||
print('DEBUG: No actor was found for ' + handle)
|
||
return 4
|
||
|
||
authHeader = createBasicAuthHeader(fromNickname, password)
|
||
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = postJson(session, newBlockJson, [], inboxUrl, headers)
|
||
if not postResult:
|
||
print('WARN: Unable to post block')
|
||
|
||
if debug:
|
||
print('DEBUG: c2s POST block success')
|
||
|
||
return newBlockJson
|
||
|
||
|
||
def sendUndoBlockViaServer(baseDir: str, session,
|
||
fromNickname: str, password: str,
|
||
fromDomain: str, fromPort: int,
|
||
httpPrefix: str, blockedUrl: str,
|
||
cachedWebfingers: {}, personCache: {},
|
||
debug: bool, projectVersion: str) -> {}:
|
||
"""Creates a block via c2s
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendBlockViaServer')
|
||
return 6
|
||
|
||
fromDomainFull = fromDomain
|
||
if fromPort:
|
||
if fromPort != 80 and fromPort != 443:
|
||
if ':' not in fromDomain:
|
||
fromDomainFull = fromDomain + ':' + str(fromPort)
|
||
|
||
toUrl = 'https://www.w3.org/ns/activitystreams#Public'
|
||
ccUrl = httpPrefix + '://' + fromDomainFull + '/users/' + \
|
||
fromNickname + '/followers'
|
||
|
||
blockActor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
|
||
newBlockJson = {
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
'type': 'Undo',
|
||
'actor': blockActor,
|
||
'object': {
|
||
'type': 'Block',
|
||
'actor': blockActor,
|
||
'object': blockedUrl,
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
}
|
||
}
|
||
|
||
handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = webfingerHandle(session, handle, httpPrefix,
|
||
cachedWebfingers,
|
||
fromDomain, projectVersion)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: announce webfinger failed for ' + handle)
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
|
||
str(wfRequest))
|
||
return 1
|
||
|
||
postToBox = 'outbox'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
fromPersonId, sharedInbox, avatarUrl,
|
||
displayName) = getPersonBox(baseDir, session, wfRequest, personCache,
|
||
projectVersion, httpPrefix, fromNickname,
|
||
fromDomain, postToBox)
|
||
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: No ' + postToBox + ' was found for ' + handle)
|
||
return 3
|
||
if not fromPersonId:
|
||
if debug:
|
||
print('DEBUG: No actor was found for ' + handle)
|
||
return 4
|
||
|
||
authHeader = createBasicAuthHeader(fromNickname, password)
|
||
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = postJson(session, newBlockJson, [], inboxUrl, headers)
|
||
if not postResult:
|
||
print('WARN: Unable to post block')
|
||
|
||
if debug:
|
||
print('DEBUG: c2s POST block success')
|
||
|
||
return newBlockJson
|
||
|
||
|
||
def postIsMuted(baseDir: str, nickname: str, domain: str,
|
||
postJsonObject: {}, messageId: str) -> bool:
|
||
""" Returns true if the given post is muted
|
||
"""
|
||
isMuted = postJsonObject.get('muted')
|
||
if isMuted is True or isMuted is False:
|
||
return isMuted
|
||
postDir = baseDir + '/accounts/' + nickname + '@' + domain
|
||
muteFilename = \
|
||
postDir + '/inbox/' + messageId.replace('/', '#') + '.json.muted'
|
||
if os.path.isfile(muteFilename):
|
||
return True
|
||
muteFilename = \
|
||
postDir + '/outbox/' + messageId.replace('/', '#') + '.json.muted'
|
||
if os.path.isfile(muteFilename):
|
||
return True
|
||
muteFilename = \
|
||
baseDir + '/accounts/cache/announce/' + nickname + \
|
||
'/' + messageId.replace('/', '#') + '.json.muted'
|
||
if os.path.isfile(muteFilename):
|
||
return True
|
||
return False
|