Support for libretranslate

daemon.py

@@ -4430,7 +4430,39 @@ class PubServer(BaseHTTPRequestHandler):
                                 setConfigParam(baseDir,
                                                'customSubmitText', '')
 
-                        # change instance description
+                        # libretranslate URL
+                        currLibretranslateUrl = \
+                            getConfigParam(baseDir,
+                                           'libretranslateUrl')
+                        if fields.get('libretranslateUrl'):
+                            if fields['libretranslateUrl'] != \
+                               currLibretranslateUrl:
+                                ltUrl = fields['libretranslateUrl']
+                                setConfigParam(baseDir,
+                                               'libretranslateUrl',
+                                               ltUrl)
+                        else:
+                            if currLibretranslateUrl:
+                                setConfigParam(baseDir,
+                                               'libretranslateUrl', '')
+
+                        # libretranslate API Key
+                        currLibretranslateApiKey = \
+                            getConfigParam(baseDir,
+                                           'libretranslateApiKey')
+                        if fields.get('libretranslateApiKey'):
+                            if fields['libretranslateApiKey'] != \
+                               currLibretranslateApiKey:
+                                ltApiKey = fields['libretranslateApiKey']
+                                setConfigParam(baseDir,
+                                               'libretranslateApiKey',
+                                               ltApiKey)
+                        else:
+                            if currLibretranslateApiKey:
+                                setConfigParam(baseDir,
+                                               'libretranslateApiKey', '')
+
+                        # change instance short description
                         currInstanceDescriptionShort = \
                             getConfigParam(baseDir,
                                            'instanceDescriptionShort')
@@ -4445,6 +4477,8 @@ class PubServer(BaseHTTPRequestHandler):
                             if currInstanceDescriptionShort:
                                 setConfigParam(baseDir,
                                                'instanceDescriptionShort', '')
+
+                        # change instance description
                         currInstanceDescription = \
                             getConfigParam(baseDir, 'instanceDescription')
                         if fields.get('instanceDescription'):

languages.py

@@ -8,7 +8,11 @@ __status__ = "Production"
 __module_group__ = "Core"
 
 import os
+import json
+from urllib import request, parse
 from utils import acctDir
+from utils import hasObjectDict
+from utils import getConfigParam
 from cache import getPersonFromCache
 
 
@@ -126,4 +130,113 @@ def understoodPostLanguage(baseDir: str, nickname: str, domain: str,
     for lang in languagesUnderstood:
         if msgObject['contentMap'].get(lang):
             return True
+    # is the language for this post supported by libretranslate?
+    libretranslateUrl = getConfigParam(baseDir, "libretranslateUrl")
+    if libretranslateUrl:
+        libretranslateApiKey = getConfigParam(baseDir, "libretranslateApiKey")
+        langList = \
+            _libretranslateLanguages(libretranslateUrl, libretranslateApiKey)
+        for lang in langList:
+            if msgObject['contentMap'].get(lang):
+                return True
     return False
+
+
+def _libretranslateLanguages(url: str, apiKey: str = None) -> []:
+    """Returns a list of supported languages
+    """
+    if not url.endswith('/languages'):
+        if not url.endswith('/'):
+            url += "/languages"
+        else:
+            url += "languages"
+
+    params = dict()
+
+    if apiKey:
+        params["api_key"] = apiKey
+
+    urlParams = parse.urlencode(params)
+
+    req = request.Request(url, data=urlParams.encode())
+
+    response = request.urlopen(req)
+
+    response_str = response.read().decode()
+
+    result = json.loads(response_str)
+    if not result:
+        return []
+    if not isinstance(result, list):
+        return []
+
+    langList = []
+    for lang in result:
+        if not isinstance(lang, dict):
+            continue
+        if not lang.get('code'):
+            continue
+        langCode = lang['code']
+        if len(langCode) != 2:
+            continue
+        langList.append(langCode)
+    langList.sort()
+    return langList
+
+
+def _libretranslate(url: str, text: str,
+                    source: str, target: str, apiKey: str = None) -> str:
+    """Translate string using libretranslate
+    """
+
+    if not url.endswith('/translate'):
+        if not url.endswith('/'):
+            url += "/translate"
+        else:
+            url += "translate"
+
+    ltParams = {
+        "q": text,
+        "source": source,
+        "target": target
+    }
+
+    if apiKey:
+        ltParams["api_key"] = apiKey
+
+    urlParams = parse.urlencode(ltParams)
+
+    req = request.Request(url, data=urlParams.encode())
+    response = request.urlopen(req)
+
+    response_str = response.read().decode()
+
+    return json.loads(response_str)["translatedText"]
+
+
+def autoTranslatePost(baseDir: str, postJsonObject: {},
+                      systemLanguage: str) -> str:
+    """Tries to automatically translate the given post
+    """
+    if not hasObjectDict(postJsonObject):
+        return ''
+    msgObject = postJsonObject['object']
+    if not msgObject.get('contentMap'):
+        return ''
+    if not isinstance(msgObject['contentMap'], dict):
+        return ''
+
+    # is the language for this post supported by libretranslate?
+    libretranslateUrl = getConfigParam(baseDir, "libretranslateUrl")
+    if not libretranslateUrl:
+        return ''
+    libretranslateApiKey = getConfigParam(baseDir, "libretranslateApiKey")
+    langList = \
+        _libretranslateLanguages(libretranslateUrl, libretranslateApiKey)
+    for lang in langList:
+        if msgObject['contentMap'].get(lang):
+            return _libretranslate(libretranslateUrl,
+                                   msgObject['contentMap']['lang'],
+                                   lang, systemLanguage,
+                                   libretranslateApiKey)
+    return ''

utils.py

@@ -30,6 +30,7 @@ invalidCharacters = (
 
 def getContentFromPost(postJsonObject: {}, systemLanguage: str) -> str:
     """Returns the content from the post in the given language
+    including searching for a matching entry within contentMap
     """
     thisPostJson = postJsonObject
     if hasObjectDict(postJsonObject):
@@ -47,6 +48,17 @@ def getContentFromPost(postJsonObject: {}, systemLanguage: str) -> str:
     return content
 
 
+def getBaseContentFromPost(postJsonObject: {}, systemLanguage: str) -> str:
+    """Returns the content from the post in the given language
+    """
+    thisPostJson = postJsonObject
+    if hasObjectDict(postJsonObject):
+        thisPostJson = postJsonObject['object']
+    if not thisPostJson.get('content'):
+        return ''
+    return thisPostJson['content']
+
+
 def acctDir(baseDir: str, nickname: str, domain: str) -> str:
     return baseDir + '/accounts/' + nickname + '@' + domain
 

webapp_post.py

@@ -22,6 +22,7 @@ from posts import postIsMuted
 from posts import getPersonBox
 from posts import downloadAnnounce
 from posts import populateRepliesJson
+from utils import getBaseContentFromPost
 from utils import getContentFromPost
 from utils import hasObjectDict
 from utils import updateAnnounceCollection
@@ -72,6 +73,7 @@ from webapp_question import insertQuestion
 from devices import E2EEdecryptMessageFromDevice
 from webfinger import webfingerHandle
 from speaker import updateSpeaker
+from languages import autoTranslatePost
 
 
 def _logPostTiming(enableTimingLog: bool, postStartTime, debugId: str) -> None:
@@ -286,7 +288,7 @@ def _getReplyIconHtml(nickname: str, isPublicRepeat: bool,
         if isinstance(postJsonObject['object']['attributedTo'], str):
             replyToLink += \
                 '?mention=' + postJsonObject['object']['attributedTo']
-    content = getContentFromPost(postJsonObject, systemLanguage)
+    content = getBaseContentFromPost(postJsonObject, systemLanguage)
     if content:
         mentionedActors = getMentionsFromHtml(content)
         if mentionedActors:
@@ -1592,7 +1594,9 @@ def individualPostAsHtml(allowDownloads: bool,
 
     contentStr = getContentFromPost(postJsonObject, systemLanguage)
     if not contentStr:
-        return ''
+        contentStr = autoTranslatePost(baseDir, postJsonObject, systemLanguage)
+        if not contentStr:
+            return ''
 
     isPatch = isGitPatch(baseDir, nickname, domain,
                          postJsonObject['object']['type'],

webapp_profile.py

@@ -1612,6 +1612,29 @@ def _htmlEditProfileChangePassword(translate: {}) -> str:
     return editProfileForm
 
 
+def _htmlEditProfileLibreTranslate(translate: {},
+                                   libretranslateUrl: str,
+                                   libretranslateApiKey: str) -> str:
+    """Change automatic translation settings
+    """
+    if libretranslateUrl is None:
+        libretranslateUrl = ''
+    if libretranslateApiKey is None:
+        libretranslateApiKey = ''
+
+    editProfileForm = \
+        '    <details><summary class="cw">LibreTranslate</summary>\n' + \
+        '    <div class="container">\n' + \
+        '<label class="labels">URL</label><br>\n' + \
+        '      <input type="text" name="libretranslateUrl" ' + \
+        'value="' + libretranslateUrl + '"><br>\n' + \
+        '<label class="labels">API Key</label><br>\n' + \
+        '      <input type="text" name="libretranslateApiKey" ' + \
+        'value="' + libretranslateApiKey + '">\n' + \
+        '    </div></details>\n'
+    return editProfileForm
+
+
 def _htmlEditProfileBackground(newsInstance: bool, translate: {}) -> str:
     """Background images section of edit profile screen
     """
@@ -2042,8 +2065,7 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str,
                                          blogsInstanceStr,
                                          newsInstanceStr)
 
-    instanceTitle = \
+    instanceTitle = getConfigParam(baseDir, 'instanceTitle')
-        getConfigParam(baseDir, 'instanceTitle')
     editProfileForm = htmlHeaderWithExternalStyle(cssFilename, instanceTitle)
 
     # keyboard navigation
@@ -2099,6 +2121,14 @@ def htmlEditProfile(cssCache: {}, translate: {}, baseDir: str, path: str,
     # Change password
     editProfileForm += _htmlEditProfileChangePassword(translate)
 
+    # automatic translations
+    libretranslateUrl = getConfigParam(baseDir, 'libretranslateUrl')
+    libretranslateApiKey = getConfigParam(baseDir, 'libretranslateApiKey')
+    editProfileForm += \
+        _htmlEditProfileLibreTranslate(translate,
+                                       libretranslateUrl,
+                                       libretranslateApiKey)
+
     # Filtering and blocking section
     editProfileForm += \
         _htmlEditProfileFiltering(baseDir, nickname, domain,