Merge

conversation.py

@@ -31,23 +31,6 @@ def _getConversationFilename(baseDir: str, nickname: str, domain: str,
     return conversationDir + '/' + conversationId
 
 
-def previousConversationPostId(baseDir: str, nickname: str, domain: str,
-                               postJsonObject: {}) -> str:
-    """Returns the previous conversation post id
-    """
-    conversationFilename = \
-        _getConversationFilename(baseDir, nickname, domain, postJsonObject)
-    if not conversationFilename:
-        return False
-    if not os.path.isfile(conversationFilename):
-        return False
-    with open(conversationFilename, 'r') as fp:
-        lines = fp.readlines()
-        if lines:
-            return lines[-1].replace('\n', '')
-    return False
-
-
 def updateConversation(baseDir: str, nickname: str, domain: str,
                        postJsonObject: {}) -> bool:
     """Ads a post to a conversation index in the /conversation subdirectory

epicyon-graph.css

@@ -0,0 +1,141 @@
+body, table, input, select, textarea {
+
+}
+
+.graph {
+    margin-bottom:1em;
+    font:normal 100%/150% arial,helvetica,sans-serif;
+}
+
+.graph caption {
+    font:bold 150%/120% arial,helvetica,sans-serif;
+    padding-bottom:0.33em;
+}
+
+.graph tbody th {
+    text-align:right;
+}
+
+@supports (display:grid) {
+
+    @media (min-width:32em) {
+
+	.graph {
+	    display:block;
+	    width:600px;
+	    height:300px;
+	}
+
+	.graph caption {
+	    display:block;
+	}
+
+	.graph thead {
+	    display:none;
+	}
+
+	.graph tbody {
+	    position:relative;
+	    display:grid;
+	    grid-template-columns:repeat(auto-fit, minmax(2em, 1fr));
+	    column-gap:2.5%;
+	    align-items:end;
+	    height:100%;
+	    margin:3em 0 1em 2.8em;
+	    padding:0 1em;
+	    border-bottom:2px solid rgba(0,0,0,0.5);
+	    background:repeating-linear-gradient(
+		180deg,
+		rgba(170,170,170,0.7) 0,
+		rgba(170,170,170,0.7) 1px,
+		transparent 1px,
+		transparent 20%
+	    );
+	}
+
+	.graph tbody:before,
+	.graph tbody:after {
+	    position:absolute;
+	    left:-3.2em;
+	    width:2.8em;
+	    text-align:right;
+	    font:bold 80%/120% arial,helvetica,sans-serif;
+	}
+
+	.graph tbody:before {
+	    content:"100%";
+	    top:-0.6em;
+	}
+
+	.graph tbody:after {
+	    content:"0%";
+	    bottom:-0.6em;
+	}
+
+	.graph tr {
+	    position:relative;
+	    display:block;
+	}
+
+	.graph tr:hover {
+	    z-index:999;
+	}
+
+	.graph th,
+	.graph td {
+	    display:block;
+	    text-align:center;
+	}
+
+	.graph tbody th {
+	    position:absolute;
+	    top:-3em;
+	    left:0;
+	    width:100%;
+	    font-weight:normal;
+	    text-align:center;
+	    white-space:nowrap;
+	    text-indent:0;
+	    transform:rotate(-45deg);
+	}
+
+	.graph tbody th:after {
+	    content:"";
+	}
+
+	.graph td {
+	    width:100%;
+	    height:100%;
+	    background:#F63;
+	    border-radius:0.5em 0.5em 0 0;
+	    transition:background 0.5s;
+	}
+
+	.graph tr:hover td {
+	    opacity:0.7;
+	}
+
+	.graph td span {
+	    overflow:hidden;
+	    position:absolute;
+	    left:50%;
+	    top:50%;
+	    width:0;
+	    padding:0.5em 0;
+	    margin:-1em 0 0;
+	    font:normal 85%/120% arial,helvetica,sans-serif;
+	    /* 			background:white; */
+	    /* 			box-shadow:0 0 0.25em rgba(0,0,0,0.6); */
+	    font-weight:bold;
+	    opacity:0;
+	    transition:opacity 0.5s;
+	    color:white;
+	}
+	.toggleGraph:checked + table td span,
+	.graph tr:hover td span {
+	    width:4em;
+	    margin-left:-2em;
+	    opacity:1;
+	}
+    }
+}

fitnessFunctions.py

@@ -0,0 +1,126 @@
+__filename__ = "fitnessFunctions.py"
+__author__ = "Bob Mottram"
+__license__ = "AGPL3+"
+__version__ = "1.2.0"
+__maintainer__ = "Bob Mottram"
+__email__ = "bob@libreserver.org"
+__status__ = "Production"
+__module_group__ = "Core"
+
+import os
+import time
+from webapp_utils import htmlHeaderWithExternalStyle
+from webapp_utils import htmlFooter
+from utils import getConfigParam
+from utils import saveJson
+
+
+def fitnessPerformance(startTime, fitnessState: {},
+                       fitnessId: str, watchPoint: str, debug: bool) -> None:
+    """Log a performance watchpoint
+    """
+    if 'performance' not in fitnessState:
+        fitnessState['performance'] = {}
+    if fitnessId not in fitnessState['performance']:
+        fitnessState['performance'][fitnessId] = {}
+    if watchPoint not in fitnessState['performance'][fitnessId]:
+        fitnessState['performance'][fitnessId][watchPoint] = {
+            "total": float(0),
+            "ctr": int(0)
+        }
+
+    timeDiff = float(time.time() - startTime)
+
+    fitnessState['performance'][fitnessId][watchPoint]['total'] += timeDiff
+    fitnessState['performance'][fitnessId][watchPoint]['ctr'] += 1
+    if fitnessState['performance'][fitnessId][watchPoint]['ctr'] >= 1024:
+        fitnessState['performance'][fitnessId][watchPoint]['total'] /= 2
+        fitnessState['performance'][fitnessId][watchPoint]['ctr'] = \
+            int(fitnessState['performance'][fitnessId][watchPoint]['ctr'] / 2)
+
+    if debug:
+        ctr = fitnessState['performance'][fitnessId][watchPoint]['ctr']
+        total = fitnessState['performance'][fitnessId][watchPoint]['total']
+        print('FITNESS: performance/' + fitnessId + '/' +
+              watchPoint + '/' + str(total * 1000 / ctr))
+
+
+def sortedWatchPoints(fitness: {}, fitnessId: str) -> []:
+    """Returns a sorted list of watchpoints
+    times are in mS
+    """
+    if not fitness.get('performance'):
+        return []
+    if not fitness['performance'].get(fitnessId):
+        return []
+    result = []
+    for watchPoint, item in fitness['performance'][fitnessId].items():
+        if not item.get('total'):
+            continue
+        averageTime = item['total'] * 1000 / item['ctr']
+        result.append(str(averageTime) + ' ' + watchPoint)
+    result.sort(reverse=True)
+    return result
+
+
+def htmlWatchPointsGraph(baseDir: str, fitness: {}, fitnessId: str,
+                         maxEntries: int) -> str:
+    """Returns the html for a graph of watchpoints
+    """
+    watchPointsList = sortedWatchPoints(fitness, fitnessId)
+
+    cssFilename = baseDir + '/epicyon-graph.css'
+    if os.path.isfile(baseDir + '/graph.css'):
+        cssFilename = baseDir + '/graph.css'
+
+    instanceTitle = \
+        getConfigParam(baseDir, 'instanceTitle')
+    htmlStr = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
+    htmlStr += \
+        '<table class="graph">\n' + \
+        '<caption>Watchpoints for ' + fitnessId + '</caption>\n' + \
+        '<thead>\n' + \
+        '  <tr>\n' + \
+        '    <th scope="col">Item</th>\n' + \
+        '    <th scope="col">Percent</th>\n' + \
+        '  </tr>\n' + \
+        '</thead><tbody>\n'
+
+    # get the maximum time
+    maxAverageTime = float(1)
+    if len(watchPointsList) > 0:
+        maxAverageTime = float(watchPointsList[0].split(' ')[0])
+    for watchPoint in watchPointsList:
+        averageTime = float(watchPoint.split(' ')[0])
+        if averageTime > maxAverageTime:
+            maxAverageTime = averageTime
+
+    ctr = 0
+    for watchPoint in watchPointsList:
+        name = watchPoint.split(' ')[1]
+        averageTime = float(watchPoint.split(' ')[0])
+        heightPercent = int(averageTime * 100 / maxAverageTime)
+        timeMS = int(averageTime)
+        if heightPercent == 0:
+            continue
+        htmlStr += \
+            '<tr style="height:' + str(heightPercent) + '%">\n' + \
+            '  <th scope="row">' + name + '</th>\n' + \
+            '  <td><span>' + str(timeMS) + 'mS</span></td>\n' + \
+            '</tr>\n'
+        ctr += 1
+        if ctr >= maxEntries:
+            break
+
+    htmlStr += '</tbody></table>\n' + htmlFooter()
+    return htmlStr
+
+
+def fitnessThread(baseDir: str, fitness: {}):
+    """Thread used to save fitness function scores
+    """
+    fitnessFilename = baseDir + '/accounts/fitness.json'
+    while True:
+        # every 10 mins
+        time.sleep(60 * 10)
+        saveJson(fitness, fitnessFilename)

inbox.py

@@ -107,6 +107,37 @@ from conversation import updateConversation
 from content import validHashTag
 
 
+def _storeLastPostId(baseDir: str, nickname: str, domain: str,
+                     postJsonObject: {}) -> None:
+    """Stores the id of the last post made by an actor
+    When a new post arrives this allows it to be compared against the last
+    to see if it is an edited post.
+    It would be great if edited posts contained a back reference id to the
+    source but we don't live in that ideal world.
+    """
+    actor = postId = None
+    if hasObjectDict(postJsonObject):
+        if postJsonObject['object'].get('attributedTo'):
+            if isinstance(postJsonObject['object']['attributedTo'], str):
+                actor = postJsonObject['object']['attributedTo']
+                postId = removeIdEnding(postJsonObject['object']['id'])
+    if not actor:
+        actor = postJsonObject['actor']
+        postId = removeIdEnding(postJsonObject['id'])
+    if not actor:
+        return
+    lastpostDir = acctDir(baseDir, nickname, domain) + '/lastpost'
+    if not os.path.isdir(lastpostDir):
+        os.mkdir(lastpostDir)
+    actorFilename = lastpostDir + '/' + actor.replace('/', '#')
+    try:
+        with open(actorFilename, 'w+') as fp:
+            fp.write(postId)
+    except BaseException:
+        print('Unable to write last post id to ' + actorFilename)
+        pass
+
+
 def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
     """Extracts hashtags from an incoming post and updates the
     relevant tags files.
@@ -2889,6 +2920,9 @@ def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
                            nickname, domain, editedFilename,
                            debug, recentPostsCache)
 
+            # store the id of the last post made by this actor
+            _storeLastPostId(baseDir, nickname, domain, postJsonObject)
+
             _inboxUpdateCalendar(baseDir, handle, postJsonObject)
 
             storeHashTags(baseDir, handleName, postJsonObject)

posts.py

@@ -86,7 +86,6 @@ from linked_data_sig import generateJsonSignature
 from petnames import resolvePetnames
 from video import convertVideoToNote
 from context import getIndividualPostContext
-from conversation import previousConversationPostId
 
 
 def isModerator(baseDir: str, nickname: str) -> bool:
@@ -2095,10 +2094,15 @@ def threadSendPost(session, postJsonStr: str, federationList: [],
         if debug:
             print('Getting postJsonString for ' + inboxUrl)
         try:
-            postResult, unauthorized = \
+            postResult, unauthorized, returnCode = \
                 postJsonString(session, postJsonStr, federationList,
                                inboxUrl, signatureHeaderJson,
                                debug)
+            if returnCode >= 500 and returnCode < 600:
+                # if an instance is returning a code which indicates that
+                # it might have a runtime error, like 503, then don't
+                # continue to post to it
+                break
             if debug:
                 print('Obtained postJsonString for ' + inboxUrl +
                       ' unauthorized: ' + str(unauthorized))
@@ -2410,12 +2414,17 @@ def sendPostViaServer(signingPrivateKeyPem: str, projectVersion: str,
         'Authorization': authHeader
     }
     postDumps = json.dumps(postJsonObject)
-    postResult = \
+    postResult, unauthorized, returnCode = \
         postJsonString(session, postDumps, [],
                        inboxUrl, headers, debug, 5, True)
     if not postResult:
         if debug:
-            print('DEBUG: POST failed for c2s to ' + inboxUrl)
+            if unauthorized:
+                print('DEBUG: POST failed for c2s to ' +
+                      inboxUrl + ' unathorized')
+            else:
+                print('DEBUG: POST failed for c2s to '
+                      + inboxUrl + ' return code ' + str(returnCode))
         return 5
 
     if debug:
@@ -4986,60 +4995,74 @@ def editedPostFilename(baseDir: str, nickname: str, domain: str,
     """
     if not hasObjectDict(postJsonObject):
         return ''
+    if not postJsonObject.get('type'):
+        return ''
+    if not postJsonObject['object'].get('type'):
+        return ''
     if not postJsonObject['object'].get('published'):
         return ''
     if not postJsonObject['object'].get('id'):
         return ''
     if not postJsonObject['object'].get('content'):
         return ''
-    prevConvPostId = \
+    if not postJsonObject['object'].get('attributedTo'):
-        previousConversationPostId(baseDir, nickname, domain,
-                                   postJsonObject)
-    if not prevConvPostId:
         return ''
-    prevConvPostFilename = \
+    if not isinstance(postJsonObject['object']['attributedTo'], str):
-        locatePost(baseDir, nickname, domain, prevConvPostId, False)
-    if not prevConvPostFilename:
         return ''
-    prevPostJsonObject = loadJson(prevConvPostFilename, 0)
+    actor = postJsonObject['object']['attributedTo']
-    if not prevPostJsonObject:
+    actorFilename = \
+        acctDir(baseDir, nickname, domain) + '/lastpost/' + \
+        actor.replace('/', '#')
+    if not os.path.isfile(actorFilename):
         return ''
-    if not hasObjectDict(prevPostJsonObject):
+    postId = removeIdEnding(postJsonObject['object']['id'])
+    lastpostId = None
+    try:
+        with open(actorFilename, 'r') as fp:
+            lastpostId = fp.read()
+    except BaseException:
         return ''
-    if not prevPostJsonObject['object'].get('published'):
+    if not lastpostId:
         return ''
-    if not prevPostJsonObject['object'].get('id'):
+    if lastpostId == postId:
         return ''
-    if not prevPostJsonObject['object'].get('content'):
+    lastpostFilename = \
+        locatePost(baseDir, nickname, domain, lastpostId, False)
+    if not lastpostFilename:
         return ''
-    if prevPostJsonObject['object']['id'] == postJsonObject['object']['id']:
+    lastpostJson = loadJson(lastpostFilename, 0)
+    if not lastpostJson:
         return ''
-    id1 = removeIdEnding(prevPostJsonObject['object']['id'])
+    if not lastpostJson.get('type'):
-    if '/' not in id1:
         return ''
-    id2 = removeIdEnding(postJsonObject['object']['id'])
+    if lastpostJson['type'] != postJsonObject['type']:
-    if '/' not in id2:
         return ''
-    ending1 = id1.split('/')[-1]
+    if not lastpostJson['object'].get('type'):
-    if not ending1:
         return ''
-    ending2 = id2.split('/')[-1]
+    if lastpostJson['object']['type'] != postJsonObject['object']['type']:
-    if not ending2:
+        return
+    if not lastpostJson['object'].get('published'):
         return ''
-    if id1.replace(ending1, '') != id2.replace(ending2, ''):
+    if not lastpostJson['object'].get('id'):
+        return ''
+    if not lastpostJson['object'].get('content'):
+        return ''
+    if not lastpostJson['object'].get('attributedTo'):
+        return ''
+    if not isinstance(lastpostJson['object']['attributedTo'], str):
         return ''
     timeDiffSeconds = \
-        secondsBetweenPublished(prevPostJsonObject['object']['published'],
+        secondsBetweenPublished(lastpostJson['object']['published'],
                                 postJsonObject['object']['published'])
     if timeDiffSeconds > maxTimeDiffSeconds:
         return ''
     if debug:
-        print(id2 + ' might be an edit of ' + id1)
+        print(postId + ' might be an edit of ' + lastpostId)
-    if wordsSimilarity(prevPostJsonObject['object']['content'],
+    if wordsSimilarity(lastpostJson['object']['content'],
                        postJsonObject['object']['content'], 10) < 70:
         return ''
-    print(id2 + ' is an edit of ' + id1)
+    print(postId + ' is an edit of ' + lastpostId)
-    return prevConvPostFilename
+    return lastpostFilename
 
 
 def getOriginalPostFromAnnounceUrl(announceUrl: str, baseDir: str,

session.py

@@ -296,7 +296,7 @@ def postJsonString(session, postJsonStr: str,
                    headers: {},
                    debug: bool,
                    timeoutSec: int = 30,
-                   quiet: bool = False) -> (bool, bool):
+                   quiet: bool = False) -> (bool, bool, int):
     """Post a json message string to the inbox of another person
     The second boolean returned is true if the send is unauthorized
     NOTE: Here we post a string rather than the original json so that
@@ -310,18 +310,18 @@ def postJsonString(session, postJsonStr: str,
     except requests.exceptions.RequestException as e:
         if not quiet:
             print('WARN: error during postJsonString requests ' + str(e))
-        return None, None
+        return None, None, 0
     except SocketError as e:
         if not quiet and e.errno == errno.ECONNRESET:
             print('WARN: connection was reset during postJsonString')
         if not quiet:
             print('ERROR: postJsonString failed ' + inboxUrl + ' ' +
                   postJsonStr + ' ' + str(headers))
-        return None, None
+        return None, None, 0
     except ValueError as e:
         if not quiet:
             print('WARN: error during postJsonString ' + str(e))
-        return None, None
+        return None, None, 0
     if postResult.status_code < 200 or postResult.status_code > 202:
         if postResult.status_code >= 400 and \
            postResult.status_code <= 405 and \
@@ -330,14 +330,14 @@ def postJsonString(session, postJsonStr: str,
                 print('WARN: Post to ' + inboxUrl +
                       ' is unauthorized. Code ' +
                       str(postResult.status_code))
-            return False, True
+            return False, True, postResult.status_code
         else:
             if not quiet:
                 print('WARN: Failed to post to ' + inboxUrl +
-                      ' with headers ' + str(headers))
+                      ' with headers ' + str(headers) +
-                print('status code ' + str(postResult.status_code))
+                      ' status code ' + str(postResult.status_code))
-            return False, False
+            return False, False, postResult.status_code
-    return True, False
+    return True, False, 0
 
 
 def postImage(session, attachImageFilename: str, federationList: [],

tests.py

@@ -1976,6 +1976,18 @@ def testSharedItemsFederation(baseDir: str) -> None:
     assert 'DFC:supplies' in catalogJson
     assert len(catalogJson.get('DFC:supplies')) == 3
 
+    # queue item removed
+    ctr = 0
+    while len([name for name in os.listdir(queuePath)
+               if os.path.isfile(os.path.join(queuePath, name))]) > 0:
+        ctr += 1
+        if ctr > 10:
+            break
+        time.sleep(1)
+
+#    assert len([name for name in os.listdir(queuePath)
+#                if os.path.isfile(os.path.join(queuePath, name))]) == 0
+
     # stop the servers
     thrAlice.kill()
     thrAlice.join()
@@ -1985,11 +1997,6 @@ def testSharedItemsFederation(baseDir: str) -> None:
     thrBob.join()
     assert thrBob.is_alive() is False
 
-    # queue item removed
-    time.sleep(4)
-    assert len([name for name in os.listdir(queuePath)
-                if os.path.isfile(os.path.join(queuePath, name))]) == 0
-
     os.chdir(baseDir)
     shutil.rmtree(baseDir + '/.tests')
     print('Testing federation of shared items between ' +
@@ -4523,6 +4530,7 @@ def _testFunctions():
         'runNewswireWatchdog',
         'runFederatedSharesWatchdog',
         'runFederatedSharesDaemon',
+        'fitnessThread',
         'threadSendPost',
         'sendToFollowers',
         'expireCache',
@@ -5743,6 +5751,16 @@ def _testWordsSimilarity() -> None:
         "The world of the electron and the webkit, the beauty of the baud"
     similarity = wordsSimilarity(content1, content2, minWords)
     assert similarity > 70
+    content1 = "<p>We&apos;re growing! </p><p>A new denizen " + \
+        "is frequenting HackBucket. You probably know him already " + \
+        "from her epic typos - but let&apos;s not spoil too much " + \
+        "\ud83d\udd2e</p>"
+    content2 = "<p>We&apos;re growing! </p><p>A new denizen " + \
+        "is frequenting HackBucket. You probably know them already " + \
+        "from their epic typos - but let&apos;s not spoil too much " + \
+        "\ud83d\udd2e</p>"
+    similarity = wordsSimilarity(content1, content2, minWords)
+    assert similarity > 80
 
 
 def runAllTests():

theme.py

@@ -104,7 +104,7 @@ def _getThemeFiles() -> []:
     return ('epicyon.css', 'login.css', 'follow.css',
             'suspended.css', 'calendar.css', 'blog.css',
             'options.css', 'search.css', 'links.css',
-            'welcome.css')
+            'welcome.css', 'graph.css')
 
 
 def isNewsThemeName(baseDir: str, themeName: str) -> bool: