indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Merge

merge-requests/30/head
Bob Mottram 2021-10-19 23:49:48 +01:00
parent 25ca2a67ec 5f6c4fac8d
commit b03a6a6104
10 changed files with 1337 additions and 777 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
architecture/epicyon_groups_ActivityPub_Security.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 95 KiB

After

Width:  |  Height:  |  Size: 101 KiB

17
conversation.py
View File

@ -31,23 +31,6 @@ def _getConversationFilename(baseDir: str, nickname: str, domain: str,
return conversationDir + '/' + conversationId
def previousConversationPostId(baseDir: str, nickname: str, domain: str,
postJsonObject: {}) -> str:
"""Returns the previous conversation post id
"""
conversationFilename = \
_getConversationFilename(baseDir, nickname, domain, postJsonObject)
if not conversationFilename:
return False
if not os.path.isfile(conversationFilename):
return False
with open(conversationFilename, 'r') as fp:
lines = fp.readlines()
if lines:
return lines[-1].replace('\n', '')
return False
def updateConversation(baseDir: str, nickname: str, domain: str,
postJsonObject: {}) -> bool:
"""Ads a post to a conversation index in the /conversation subdirectory

1661
daemon.py
View File

File diff suppressed because it is too large Load Diff

141
epicyon-graph.css 100644
View File

@ -0,0 +1,141 @@
body, table, input, select, textarea {
}
.graph {
margin-bottom:1em;
font:normal 100%/150% arial,helvetica,sans-serif;
}
.graph caption {
font:bold 150%/120% arial,helvetica,sans-serif;
padding-bottom:0.33em;
}
.graph tbody th {
text-align:right;
}
@supports (display:grid) {
@media (min-width:32em) {
.graph {
display:block;
width:600px;
height:300px;
}
.graph caption {
display:block;
}
.graph thead {
display:none;
}
.graph tbody {
position:relative;
display:grid;
grid-template-columns:repeat(auto-fit, minmax(2em, 1fr));
column-gap:2.5%;
align-items:end;
height:100%;
margin:3em 0 1em 2.8em;
padding:0 1em;
border-bottom:2px solid rgba(0,0,0,0.5);
background:repeating-linear-gradient(
180deg,
rgba(170,170,170,0.7) 0,
rgba(170,170,170,0.7) 1px,
transparent 1px,
transparent 20%
);
}
.graph tbody:before,
.graph tbody:after {
position:absolute;
left:-3.2em;
width:2.8em;
text-align:right;
font:bold 80%/120% arial,helvetica,sans-serif;
}
.graph tbody:before {
content:"100%";
top:-0.6em;
}
.graph tbody:after {
content:"0%";
bottom:-0.6em;
}
.graph tr {
position:relative;
display:block;
}
.graph tr:hover {
z-index:999;
}
.graph th,
.graph td {
display:block;
text-align:center;
}
.graph tbody th {
position:absolute;
top:-3em;
left:0;
width:100%;
font-weight:normal;
text-align:center;
white-space:nowrap;
text-indent:0;
transform:rotate(-45deg);
}
.graph tbody th:after {
content:"";
}
.graph td {
width:100%;
height:100%;
background:#F63;
border-radius:0.5em 0.5em 0 0;
transition:background 0.5s;
}
.graph tr:hover td {
opacity:0.7;
}
.graph td span {
overflow:hidden;
position:absolute;
left:50%;
top:50%;
width:0;
padding:0.5em 0;
margin:-1em 0 0;
font:normal 85%/120% arial,helvetica,sans-serif;
/* background:white; */
/* box-shadow:0 0 0.25em rgba(0,0,0,0.6); */
font-weight:bold;
opacity:0;
transition:opacity 0.5s;
color:white;
}
.toggleGraph:checked + table td span,
.graph tr:hover td span {
width:4em;
margin-left:-2em;
opacity:1;
}
}
}

126
fitnessFunctions.py 100644
View File

@ -0,0 +1,126 @@
__filename__ = "fitnessFunctions.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.2.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@libreserver.org"
__status__ = "Production"
__module_group__ = "Core"
import os
import time
from webapp_utils import htmlHeaderWithExternalStyle
from webapp_utils import htmlFooter
from utils import getConfigParam
from utils import saveJson
def fitnessPerformance(startTime, fitnessState: {},
fitnessId: str, watchPoint: str, debug: bool) -> None:
"""Log a performance watchpoint
"""
if 'performance' not in fitnessState:
fitnessState['performance'] = {}
if fitnessId not in fitnessState['performance']:
fitnessState['performance'][fitnessId] = {}
if watchPoint not in fitnessState['performance'][fitnessId]:
fitnessState['performance'][fitnessId][watchPoint] = {
"total": float(0),
"ctr": int(0)
}
timeDiff = float(time.time() - startTime)
fitnessState['performance'][fitnessId][watchPoint]['total'] += timeDiff
fitnessState['performance'][fitnessId][watchPoint]['ctr'] += 1
if fitnessState['performance'][fitnessId][watchPoint]['ctr'] >= 1024:
fitnessState['performance'][fitnessId][watchPoint]['total'] /= 2
fitnessState['performance'][fitnessId][watchPoint]['ctr'] = \
int(fitnessState['performance'][fitnessId][watchPoint]['ctr'] / 2)
if debug:
ctr = fitnessState['performance'][fitnessId][watchPoint]['ctr']
total = fitnessState['performance'][fitnessId][watchPoint]['total']
print('FITNESS: performance/' + fitnessId + '/' +
watchPoint + '/' + str(total * 1000 / ctr))
def sortedWatchPoints(fitness: {}, fitnessId: str) -> []:
"""Returns a sorted list of watchpoints
times are in mS
"""
if not fitness.get('performance'):
return []
if not fitness['performance'].get(fitnessId):
return []
result = []
for watchPoint, item in fitness['performance'][fitnessId].items():
if not item.get('total'):
continue
averageTime = item['total'] * 1000 / item['ctr']
result.append(str(averageTime) + ' ' + watchPoint)
result.sort(reverse=True)
return result
def htmlWatchPointsGraph(baseDir: str, fitness: {}, fitnessId: str,
maxEntries: int) -> str:
"""Returns the html for a graph of watchpoints
"""
watchPointsList = sortedWatchPoints(fitness, fitnessId)
cssFilename = baseDir + '/epicyon-graph.css'
if os.path.isfile(baseDir + '/graph.css'):
cssFilename = baseDir + '/graph.css'
instanceTitle = \
getConfigParam(baseDir, 'instanceTitle')
htmlStr = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
htmlStr += \
'<table class="graph">\n' + \
'<caption>Watchpoints for ' + fitnessId + '</caption>\n' + \
'<thead>\n' + \
' <tr>\n' + \
' <th scope="col">Item</th>\n' + \
' <th scope="col">Percent</th>\n' + \
' </tr>\n' + \
'</thead><tbody>\n'
# get the maximum time
maxAverageTime = float(1)
if len(watchPointsList) > 0:
maxAverageTime = float(watchPointsList[0].split(' ')[0])
for watchPoint in watchPointsList:
averageTime = float(watchPoint.split(' ')[0])
if averageTime > maxAverageTime:
maxAverageTime = averageTime
ctr = 0
for watchPoint in watchPointsList:
name = watchPoint.split(' ')[1]
averageTime = float(watchPoint.split(' ')[0])
heightPercent = int(averageTime * 100 / maxAverageTime)
timeMS = int(averageTime)
if heightPercent == 0:
continue
htmlStr += \
'<tr style="height:' + str(heightPercent) + '%">\n' + \
' <th scope="row">' + name + '</th>\n' + \
' <td><span>' + str(timeMS) + 'mS</span></td>\n' + \
'</tr>\n'
ctr += 1
if ctr >= maxEntries:
break
htmlStr += '</tbody></table>\n' + htmlFooter()
return htmlStr
def fitnessThread(baseDir: str, fitness: {}):
"""Thread used to save fitness function scores
"""
fitnessFilename = baseDir + '/accounts/fitness.json'
while True:
# every 10 mins
time.sleep(60 * 10)
saveJson(fitness, fitnessFilename)

34
inbox.py
View File

@ -107,6 +107,37 @@ from conversation import updateConversation
from content import validHashTag
def _storeLastPostId(baseDir: str, nickname: str, domain: str,
postJsonObject: {}) -> None:
"""Stores the id of the last post made by an actor
When a new post arrives this allows it to be compared against the last
to see if it is an edited post.
It would be great if edited posts contained a back reference id to the
source but we don't live in that ideal world.
"""
actor = postId = None
if hasObjectDict(postJsonObject):
if postJsonObject['object'].get('attributedTo'):
if isinstance(postJsonObject['object']['attributedTo'], str):
actor = postJsonObject['object']['attributedTo']
postId = removeIdEnding(postJsonObject['object']['id'])
if not actor:
actor = postJsonObject['actor']
postId = removeIdEnding(postJsonObject['id'])
if not actor:
return
lastpostDir = acctDir(baseDir, nickname, domain) + '/lastpost'
if not os.path.isdir(lastpostDir):
os.mkdir(lastpostDir)
actorFilename = lastpostDir + '/' + actor.replace('/', '#')
try:
with open(actorFilename, 'w+') as fp:
fp.write(postId)
except BaseException:
print('Unable to write last post id to ' + actorFilename)
pass
def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
"""Extracts hashtags from an incoming post and updates the
relevant tags files.
@ -2889,6 +2920,9 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
nickname, domain, editedFilename,
debug, recentPostsCache)
# store the id of the last post made by this actor
_storeLastPostId(baseDir, nickname, domain, postJsonObject)
_inboxUpdateCalendar(baseDir, handle, postJsonObject)
storeHashTags(baseDir, handleName, postJsonObject)

87
posts.py
View File

@ -86,7 +86,6 @@ from linked_data_sig import generateJsonSignature
from petnames import resolvePetnames
from video import convertVideoToNote
from context import getIndividualPostContext
from conversation import previousConversationPostId
def isModerator(baseDir: str, nickname: str) -> bool:
@ -2095,10 +2094,15 @@ def threadSendPost(session, postJsonStr: str, federationList: [],
if debug:
print('Getting postJsonString for ' + inboxUrl)
try:
postResult, unauthorized = \
postResult, unauthorized, returnCode = \
postJsonString(session, postJsonStr, federationList,
inboxUrl, signatureHeaderJson,
debug)
if returnCode >= 500 and returnCode < 600:
# if an instance is returning a code which indicates that
# it might have a runtime error, like 503, then don't
# continue to post to it
break
if debug:
print('Obtained postJsonString for ' + inboxUrl +
' unauthorized: ' + str(unauthorized))
@ -2410,12 +2414,17 @@ def sendPostViaServer(signingPrivateKeyPem: str, projectVersion: str,
'Authorization': authHeader
}
postDumps = json.dumps(postJsonObject)
postResult = \
postResult, unauthorized, returnCode = \
postJsonString(session, postDumps, [],
inboxUrl, headers, debug, 5, True)
if not postResult:
if debug:
print('DEBUG: POST failed for c2s to ' + inboxUrl)
if unauthorized:
print('DEBUG: POST failed for c2s to ' +
inboxUrl + ' unathorized')
else:
print('DEBUG: POST failed for c2s to '
+ inboxUrl + ' return code ' + str(returnCode))
return 5
if debug:
@ -4986,60 +4995,74 @@ def editedPostFilename(baseDir: str, nickname: str, domain: str,
"""
if not hasObjectDict(postJsonObject):
return ''
if not postJsonObject.get('type'):
return ''
if not postJsonObject['object'].get('type'):
return ''
if not postJsonObject['object'].get('published'):
return ''
if not postJsonObject['object'].get('id'):
return ''
if not postJsonObject['object'].get('content'):
return ''
prevConvPostId = \
previousConversationPostId(baseDir, nickname, domain,
postJsonObject)
if not prevConvPostId:
if not postJsonObject['object'].get('attributedTo'):
return ''
prevConvPostFilename = \
locatePost(baseDir, nickname, domain, prevConvPostId, False)
if not prevConvPostFilename:
if not isinstance(postJsonObject['object']['attributedTo'], str):
return ''
prevPostJsonObject = loadJson(prevConvPostFilename, 0)
if not prevPostJsonObject:
actor = postJsonObject['object']['attributedTo']
actorFilename = \
acctDir(baseDir, nickname, domain) + '/lastpost/' + \
actor.replace('/', '#')
if not os.path.isfile(actorFilename):
return ''
if not hasObjectDict(prevPostJsonObject):
postId = removeIdEnding(postJsonObject['object']['id'])
lastpostId = None
try:
with open(actorFilename, 'r') as fp:
lastpostId = fp.read()
except BaseException:
return ''
if not prevPostJsonObject['object'].get('published'):
if not lastpostId:
return ''
if not prevPostJsonObject['object'].get('id'):
if lastpostId == postId:
return ''
if not prevPostJsonObject['object'].get('content'):
lastpostFilename = \
locatePost(baseDir, nickname, domain, lastpostId, False)
if not lastpostFilename:
return ''
if prevPostJsonObject['object']['id'] == postJsonObject['object']['id']:
lastpostJson = loadJson(lastpostFilename, 0)
if not lastpostJson:
return ''
id1 = removeIdEnding(prevPostJsonObject['object']['id'])
if '/' not in id1:
if not lastpostJson.get('type'):
return ''
id2 = removeIdEnding(postJsonObject['object']['id'])
if '/' not in id2:
if lastpostJson['type'] != postJsonObject['type']:
return ''
ending1 = id1.split('/')[-1]
if not ending1:
if not lastpostJson['object'].get('type'):
return ''
ending2 = id2.split('/')[-1]
if not ending2:
if lastpostJson['object']['type'] != postJsonObject['object']['type']:
return
if not lastpostJson['object'].get('published'):
return ''
if id1.replace(ending1, '') != id2.replace(ending2, ''):
if not lastpostJson['object'].get('id'):
return ''
if not lastpostJson['object'].get('content'):
return ''
if not lastpostJson['object'].get('attributedTo'):
return ''
if not isinstance(lastpostJson['object']['attributedTo'], str):
return ''
timeDiffSeconds = \
secondsBetweenPublished(prevPostJsonObject['object']['published'],
secondsBetweenPublished(lastpostJson['object']['published'],
postJsonObject['object']['published'])
if timeDiffSeconds > maxTimeDiffSeconds:
return ''
if debug:
print(id2 + ' might be an edit of ' + id1)
if wordsSimilarity(prevPostJsonObject['object']['content'],
print(postId + ' might be an edit of ' + lastpostId)
if wordsSimilarity(lastpostJson['object']['content'],
postJsonObject['object']['content'], 10) < 70:
return ''
print(id2 + ' is an edit of ' + id1)
return prevConvPostFilename
print(postId + ' is an edit of ' + lastpostId)
return lastpostFilename
def getOriginalPostFromAnnounceUrl(announceUrl: str, baseDir: str,

18
session.py
View File

@ -296,7 +296,7 @@ def postJsonString(session, postJsonStr: str,
headers: {},
debug: bool,
timeoutSec: int = 30,
quiet: bool = False) -> (bool, bool):
quiet: bool = False) -> (bool, bool, int):
"""Post a json message string to the inbox of another person
The second boolean returned is true if the send is unauthorized
NOTE: Here we post a string rather than the original json so that
@ -310,18 +310,18 @@ def postJsonString(session, postJsonStr: str,
except requests.exceptions.RequestException as e:
if not quiet:
print('WARN: error during postJsonString requests ' + str(e))
return None, None
return None, None, 0
except SocketError as e:
if not quiet and e.errno == errno.ECONNRESET:
print('WARN: connection was reset during postJsonString')
if not quiet:
print('ERROR: postJsonString failed ' + inboxUrl + ' ' +
postJsonStr + ' ' + str(headers))
return None, None
return None, None, 0
except ValueError as e:
if not quiet:
print('WARN: error during postJsonString ' + str(e))
return None, None
return None, None, 0
if postResult.status_code < 200 or postResult.status_code > 202:
if postResult.status_code >= 400 and \
postResult.status_code <= 405 and \
@ -330,14 +330,14 @@ def postJsonString(session, postJsonStr: str,
print('WARN: Post to ' + inboxUrl +
' is unauthorized. Code ' +
str(postResult.status_code))
return False, True
return False, True, postResult.status_code
else:
if not quiet:
print('WARN: Failed to post to ' + inboxUrl +
' with headers ' + str(headers))
print('status code ' + str(postResult.status_code))
return False, False
return True, False
' with headers ' + str(headers) +
' status code ' + str(postResult.status_code))
return False, False, postResult.status_code
return True, False, 0
def postImage(session, attachImageFilename: str, federationList: [],

28
tests.py
View File

@ -1976,6 +1976,18 @@ def testSharedItemsFederation(baseDir: str) -> None:
assert 'DFC:supplies' in catalogJson
assert len(catalogJson.get('DFC:supplies')) == 3
# queue item removed
ctr = 0
while len([name for name in os.listdir(queuePath)
if os.path.isfile(os.path.join(queuePath, name))]) > 0:
ctr += 1
if ctr > 10:
break
time.sleep(1)
# assert len([name for name in os.listdir(queuePath)
# if os.path.isfile(os.path.join(queuePath, name))]) == 0
# stop the servers
thrAlice.kill()
thrAlice.join()
@ -1985,11 +1997,6 @@ def testSharedItemsFederation(baseDir: str) -> None:
thrBob.join()
assert thrBob.is_alive() is False
# queue item removed
time.sleep(4)
assert len([name for name in os.listdir(queuePath)
if os.path.isfile(os.path.join(queuePath, name))]) == 0
os.chdir(baseDir)
shutil.rmtree(baseDir + '/.tests')
print('Testing federation of shared items between ' +
@ -4523,6 +4530,7 @@ def _testFunctions():
'runNewswireWatchdog',
'runFederatedSharesWatchdog',
'runFederatedSharesDaemon',
'fitnessThread',
'threadSendPost',
'sendToFollowers',
'expireCache',
@ -5743,6 +5751,16 @@ def _testWordsSimilarity() -> None:
"The world of the electron and the webkit, the beauty of the baud"
similarity = wordsSimilarity(content1, content2, minWords)
assert similarity > 70
content1 = "<p>We&apos;re growing! </p><p>A new denizen " + \
"is frequenting HackBucket. You probably know him already " + \
"from her epic typos - but let&apos;s not spoil too much " + \
"\ud83d\udd2e</p>"
content2 = "<p>We&apos;re growing! </p><p>A new denizen " + \
"is frequenting HackBucket. You probably know them already " + \
"from their epic typos - but let&apos;s not spoil too much " + \
"\ud83d\udd2e</p>"
similarity = wordsSimilarity(content1, content2, minWords)
assert similarity > 80
def runAllTests():

2
theme.py
View File

@ -104,7 +104,7 @@ def _getThemeFiles() -> []:
return ('epicyon.css', 'login.css', 'follow.css',
'suspended.css', 'calendar.css', 'blog.css',
'options.css', 'search.css', 'links.css',
'welcome.css')
'welcome.css', 'graph.css')
def isNewsThemeName(baseDir: str, themeName: str) -> bool: