Tidying

daemon.py

@@ -1666,7 +1666,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self._write(msg)
                                 else:
                                     if self._fetchAuthenticated():
-                                        msg=json.dumps(postJsonObject).encode('utf-8')
+                                        msg=json.dumps(postJsonObject,ensure_ascii=False).encode('utf-8')
                                         self._set_headers('application/json',len(msg),None)
                                         self._write(msg)
                                     else:
@@ -1733,7 +1733,7 @@ class PubServer(BaseHTTPRequestHandler):
                                         self._write(msg)
                                     else:
                                         if self._fetchAuthenticated():
-                                            msg=json.dumps(repliesJson).encode('utf-8')
+                                            msg=json.dumps(repliesJson,ensure_ascii=False).encode('utf-8')
                                             self._set_headers('application/json',len(msg),None)
                                             self._write(msg)
                                         else:
@@ -1782,7 +1782,7 @@ class PubServer(BaseHTTPRequestHandler):
                                         self._write(msg)
                                     else:
                                         if self._fetchAuthenticated():
-                                            msg=json.dumps(repliesJson).encode('utf-8')
+                                            msg=json.dumps(repliesJson,ensure_ascii=False).encode('utf-8')
                                             self._set_headers('application/json',len(msg),None)
                                             self._write(msg)
                                         else:
@@ -1824,7 +1824,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self._write(msg)
                             else:
                                 if self._fetchAuthenticated():
-                                    msg=json.dumps(actorJson['roles']).encode('utf-8')
+                                    msg=json.dumps(actorJson['roles'],ensure_ascii=False).encode('utf-8')
                                     self._set_headers('application/json',len(msg),None)
                                     self._write(msg)
                                 else:
@@ -1867,7 +1867,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self._write(msg)
                             else:
                                 if self._fetchAuthenticated():
-                                    msg=json.dumps(actorJson['skills']).encode('utf-8')
+                                    msg=json.dumps(actorJson['skills'],ensure_ascii=False).encode('utf-8')
                                     self._set_headers('application/json',len(msg),None)
                                     self._write(msg)
                                 else:
@@ -1923,7 +1923,7 @@ class PubServer(BaseHTTPRequestHandler):
                                     self._write(msg)
                                 else:
                                     if self._fetchAuthenticated():
-                                        msg=json.dumps(postJsonObject).encode('utf-8')
+                                        msg=json.dumps(postJsonObject,ensure_ascii=False).encode('utf-8')
                                         self._set_headers('application/json',len(msg),None)
                                         self._write(msg)
                                     else:
@@ -1986,7 +1986,7 @@ class PubServer(BaseHTTPRequestHandler):
                         else:
                             # don't need authenticated fetch here because there is
                             # already the authorization check
-                            msg=json.dumps(inboxFeed).encode('utf-8')
+                            msg=json.dumps(inboxFeed,ensure_ascii=False).encode('utf-8')
                             self._set_headers('application/json',len(msg),None)
                             self._write(msg)
                         self.server.GETbusy=False
@@ -2056,7 +2056,7 @@ class PubServer(BaseHTTPRequestHandler):
                         else:
                             # don't need authenticated fetch here because there is
                             # already the authorization check
-                            msg=json.dumps(inboxDMFeed).encode('utf-8')
+                            msg=json.dumps(inboxDMFeed,ensure_ascii=False).encode('utf-8')
                             self._set_headers('application/json',len(msg),None)
                             self._write(msg)
                         self.server.GETbusy=False
@@ -2129,7 +2129,7 @@ class PubServer(BaseHTTPRequestHandler):
                     else:
                         # don't need authenticated fetch here because there is
                         # already the authorization check
-                        msg=json.dumps(inboxRepliesFeed).encode('utf-8')
+                        msg=json.dumps(inboxRepliesFeed,ensure_ascii=False).encode('utf-8')
                         self._set_headers('application/json',len(msg),None)
                         self._write(msg)
                     self.server.GETbusy=False
@@ -2202,7 +2202,7 @@ class PubServer(BaseHTTPRequestHandler):
                     else:
                         # don't need authenticated fetch here because there is
                         # already the authorization check
-                        msg=json.dumps(inboxMediaFeed).encode('utf-8')
+                        msg=json.dumps(inboxMediaFeed,ensure_ascii=False).encode('utf-8')
                         self._set_headers('application/json',len(msg),None)
                         self._write(msg)
                     self.server.GETbusy=False
@@ -2307,7 +2307,7 @@ class PubServer(BaseHTTPRequestHandler):
                 self._write(msg)
             else:
                 if self._fetchAuthenticated():
-                    msg=json.dumps(outboxFeed).encode('utf-8')
+                    msg=json.dumps(outboxFeed,ensure_ascii=False).encode('utf-8')
                     self._set_headers('application/json',len(msg),None)
                     self._write(msg)
                 else:
@@ -2370,7 +2370,7 @@ class PubServer(BaseHTTPRequestHandler):
                         else:
                             # don't need authenticated fetch here because there is
                             # already the authorization check
-                            msg=json.dumps(moderationFeed).encode('utf-8')
+                            msg=json.dumps(moderationFeed,ensure_ascii=False).encode('utf-8')
                             self._set_headers('application/json',len(msg),None)
                             self._write(msg)
                         self.server.GETbusy=False
@@ -2440,7 +2440,7 @@ class PubServer(BaseHTTPRequestHandler):
                     return
             else:
                 if self._fetchAuthenticated():
-                    msg=json.dumps(shares).encode('utf-8')
+                    msg=json.dumps(shares,ensure_ascii=False).encode('utf-8')
                     self._set_headers('application/json',len(msg),None)
                     self._write(msg)
                 else:
@@ -2496,7 +2496,7 @@ class PubServer(BaseHTTPRequestHandler):
                     return
             else:
                 if self._fetchAuthenticated():
-                    msg=json.dumps(following).encode('utf-8')
+                    msg=json.dumps(following,ensure_ascii=False).encode('utf-8')
                     self._set_headers('application/json',len(msg),None)
                     self._write(msg)
                 else:
@@ -2551,7 +2551,7 @@ class PubServer(BaseHTTPRequestHandler):
                     return
             else:
                 if self._fetchAuthenticated():
-                    msg=json.dumps(followers).encode('utf-8')
+                    msg=json.dumps(followers,ensure_ascii=False).encode('utf-8')
                     self._set_headers('application/json',len(msg),None)
                     self._write(msg)
                 else:
@@ -2584,7 +2584,7 @@ class PubServer(BaseHTTPRequestHandler):
                 self._write(msg)
             else:
                 if self._fetchAuthenticated():
-                    msg=json.dumps(getPerson).encode('utf-8')
+                    msg=json.dumps(getPerson,ensure_ascii=False).encode('utf-8')
                     self._set_headers('application/json',len(msg),None)
                     self._write(msg)
                 else:
@@ -2610,7 +2610,7 @@ class PubServer(BaseHTTPRequestHandler):
             with open(filename, 'r', encoding='utf-8') as File:
                 content = File.read()
                 contentJson=json.loads(content)
-                msg=json.dumps(contentJson).encode('utf-8')
+                msg=json.dumps(contentJson,ensure_ascii=False).encode('utf-8')
                 self._set_headers('application/json',len(msg),None)
                 self._write(msg)
         else:
@@ -2653,10 +2653,18 @@ class PubServer(BaseHTTPRequestHandler):
             # why don't we just use msg.is_multipart(), rather than splitting?
             # TL;DR it doesn't work for this use case because we're not using
             # email style encoding message/rfc822
-            messageFields=msg.get_payload(decode=False).split(boundary)
+            imageBoundary=b'Content-Disposition: form-data; name="attachpic";'
+            imageLocation=postBytes.find(imageBoundary)
+            if imageLocation>-1:
+                # get the first part of the data containing text fields
+                # If we try to use decode=True on the full data, including images,
+                # then it will fail
+                msg = email.parser.BytesParser().parsebytes(postBytes[:imageLocation])
+            messageFields=msg.get_payload(decode=True).decode('utf-8').split(boundary)
             fields={}
             filename=None
             attachmentMediaType=None
+            # get the text fields
             for f in messageFields:
                 if f=='--':
                     continue
@@ -2675,52 +2683,56 @@ class PubServer(BaseHTTPRequestHandler):
                                             postValue+='\n'
                                         postValue+=postLines[line]
                                 fields[postKey]=postValue
-                        else:
-                            # directly search the binary array for the beginning
-                            # of an image
-                            extensionList=['png','jpeg','gif','mp4','webm','ogv','mp3','ogg']
-                            for extension in extensionList:
-                                searchStr=b'Content-Type: image/png'
-                                if extension=='jpeg':
-                                    searchStr=b'Content-Type: image/jpeg'
-                                elif extension=='gif':
-                                    searchStr=b'Content-Type: image/gif'
-                                elif extension=='mp4':
-                                    searchStr=b'Content-Type: video/mp4'
-                                elif extension=='ogv':
-                                    searchStr=b'Content-Type: video/ogv'
-                                elif extension=='mp3':
-                                    searchStr=b'Content-Type: audio/mpeg'
-                                elif extension=='ogg':
-                                    searchStr=b'Content-Type: audio/ogg'
-                                imageLocation=postBytes.find(searchStr)
-                                filenameBase= \
-                                    self.server.baseDir+'/accounts/'+ \
-                                    nickname+'@'+self.server.domain+'/upload'
-                                if imageLocation>-1:
-                                    if extension=='jpeg':
-                                        extension='jpg'
-                                    if extension=='mpeg':
-                                        extension='mp3'
-                                    filename=filenameBase+'.'+extension
-                                    attachmentMediaType= \
-                                        searchStr.decode().split('/')[0].replace('Content-Type: ','')
-                                    break
-                            if filename and imageLocation>-1:
-                                # locate the beginning of the image, after any
-                                # carriage returns
-                                startPos=imageLocation+len(searchStr)
-                                for offset in range(1,8):
-                                    if postBytes[startPos+offset]!=10:
-                                        if postBytes[startPos+offset]!=13:
-                                            startPos+=offset
-                                            break
+            # now extract any attached image or other media
+            if imageLocation>-1:
+                imageLocation2=-1
+                filename=None
+                searchStr=''
+                # directly search the binary array for the beginning
+                # of an image
+                extensionList=['png','jpeg','gif','mp4','webm','ogv','mp3','ogg']
+                for extension in extensionList:
+                    searchStr=b'Content-Type: image/png'
+                    if extension=='jpeg':
+                        searchStr=b'Content-Type: image/jpeg'
+                    elif extension=='gif':
+                        searchStr=b'Content-Type: image/gif'
+                    elif extension=='mp4':
+                        searchStr=b'Content-Type: video/mp4'
+                    elif extension=='ogv':
+                        searchStr=b'Content-Type: video/ogv'
+                    elif extension=='mp3':
+                        searchStr=b'Content-Type: audio/mpeg'
+                    elif extension=='ogg':
+                        searchStr=b'Content-Type: audio/ogg'
+                    imageLocation2=postBytes.find(searchStr)
+                    filenameBase= \
+                        self.server.baseDir+'/accounts/'+ \
+                        nickname+'@'+self.server.domain+'/upload'
+                    if imageLocation2>-1:
+                        if extension=='jpeg':
+                            extension='jpg'
+                        elif extension=='mpeg':
+                            extension='mp3'
+                        filename=filenameBase+'.'+extension
+                        attachmentMediaType= \
+                            searchStr.decode().split('/')[0].replace('Content-Type: ','')
+                        break
+                if filename and imageLocation2>-1:
+                    # locate the beginning of the image, after any
+                    # carriage returns
+                    startPos=imageLocation2+len(searchStr)
+                    for offset in range(1,8):
+                        if postBytes[startPos+offset]!=10:
+                            if postBytes[startPos+offset]!=13:
+                                startPos+=offset
+                                break
 
-                                fd = open(filename, 'wb')
-                                fd.write(postBytes[startPos:])
-                                fd.close()
-                            else:
-                                filename=None
+                    fd = open(filename, 'wb')
+                    fd.write(postBytes[startPos:])
+                    fd.close()
+                else:
+                    filename=None
 
             # send the post
             if not fields.get('message') and not fields.get('imageDescription'):
@@ -3118,6 +3130,7 @@ class PubServer(BaseHTTPRequestHandler):
                 postBytes=self.rfile.read(length)
 
                 msg = email.parser.BytesParser().parsebytes(postBytes)
+                #messageFields=msg.get_payload(decode=True).decode('utf-8').split(boundary)
                 messageFields=msg.get_payload(decode=False).split(boundary)
                 fields={}
                 filename=None

tests.py

@@ -929,6 +929,7 @@ def testFollowBetweenServers():
     assert 'bob@'+bobDomain in open(aliceDir+'/accounts/alice@'+aliceDomain+'/following.txt').read()
 
     # queue item removed
+    time.sleep(4)
     assert len([name for name in os.listdir(queuePath) if os.path.isfile(os.path.join(queuePath, name))])==0
     
     os.chdir(baseDir)
@@ -1597,8 +1598,26 @@ def testGetStatusNumber():
             assert int(statusNumber) > prevStatusNumber
         prevStatusNumber=int(statusNumber)
 
+def testCommentJson() -> None:
+    print('testCommentJson')
+    filename='/tmp/test.json'
+    messageStr="Crème brûlée यह एक परीक्षण ह"
+    testJson={
+        "content": messageStr
+    }
+    with open(filename, 'w') as fp:
+        commentjson.dump(testJson, fp, indent=2, sort_keys=False)
+    receivedJson=None
+    with open(filename, 'r') as fp:
+        receivedJson=commentjson.load(fp)
+    assert receivedJson
+    assert receivedJson['content']==messageStr
+    encodedStr=json.dumps(testJson, ensure_ascii=False)
+    assert messageStr in encodedStr
+
 def runAllTests():
     print('Running tests...')
+    testCommentJson()
     testGetStatusNumber()
     testAddEmoji()
     testWebLinks()

webinterface.py

@@ -606,7 +606,7 @@ def htmlEditProfile(translate: {},baseDir: str,path: str,domain: str,port: int)
         
     editProfileForm=htmlHeader(cssFilename,editProfileCSS)
     editProfileForm+= \
-        '<form enctype="multipart/form-data" method="POST" action="'+path+'/profiledata">' \
+        '<form enctype="multipart/form-data" method="POST" accept-charset="UTF-8" action="'+path+'/profiledata">' \
         '  <div class="vertical-center">' \
         '    <p class="new-post-text">'+translate['Profile for']+' '+nickname+'@'+domainFull+'</p>' \
         '    <div class="container">' \
@@ -1001,7 +1001,7 @@ def htmlNewPost(translate: {},baseDir: str, \
         mentionsStr='Re: '+reportUrl+'\n\n'+mentionsStr
     
     newPostForm+= \
-        '<form enctype="multipart/form-data" method="POST" action="'+path+'?'+endpoint+'?page='+str(pageNumber)+'">' \
+        '<form enctype="multipart/form-data" method="POST" accept-charset="UTF-8" action="'+path+'?'+endpoint+'?page='+str(pageNumber)+'">' \
         '  <div class="vertical-center">' \
         '    <label for="nickname"><b>'+newPostText+'</b></label>' \
         '    <div class="container">' \
@@ -3321,7 +3321,7 @@ def htmlSearch(translate: {}, \
     followStr+='  <center>'    
     followStr+='  <p class="followText">'+translate['Enter an address, shared item, #hashtag, *skill or :emoji: to search for']+'</p>'
     followStr+= \
-        '  <form method="POST" action="'+actor+'/searchhandle">' \
+        '  <form method="POST" accept-charset="UTF-8" action="'+actor+'/searchhandle">' \
         '    <input type="hidden" name="actor" value="'+actor+'">' \
         '    <input type="text" name="searchtext" autofocus><br>' \
         '    <button type="submit" class="button" name="submitSearch">'+translate['Submit']+'</button>' \