mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

flake8 style

main
Bob Mottram 2020-04-02 09:56:17 +00:00
parent a70793b616
commit dbcfbd1a8c
1 changed files with 302 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
content.py
View File

@ -7,17 +7,17 @@ __email__="bob@freedombone.net"
__status__ = "Production" __status__ = "Production"
import os import os
import time
import email.parser import email.parser
from shutil import copyfile from shutil import copyfile
from utils import loadJson from utils import loadJson
from utils import fileLastModified from utils import fileLastModified
def switchWords(baseDir: str, nickname: str, domain: str, content: str) -> str: def switchWords(baseDir: str, nickname: str, domain: str, content: str) -> str:
"""Performs word replacements. eg. Trump -> The Orange Menace """Performs word replacements. eg. Trump -> The Orange Menace
""" """
switchWordsFilename= \ switchWordsFilename = baseDir + '/accounts/' + \
baseDir+'/accounts/'+nickname+'@'+domain+'/replacewords.txt' nickname + '@' + domain + '/replacewords.txt'
if not os.path.isfile(switchWordsFilename): if not os.path.isfile(switchWordsFilename):
return content return content
with open(switchWordsFilename, 'r') as fp: with open(switchWordsFilename, 'r') as fp:
@ -37,11 +37,12 @@ def switchWords(baseDir: str,nickname: str,domain: str,content: str) -> str:
if not wordTransform: if not wordTransform:
continue continue
if len(wordTransform) == 2: if len(wordTransform) == 2:
content= \ replaceStr1 = wordTransform[0].strip().replace('"', '')
content.replace(wordTransform[0].strip().replace('"',''), \ replaceStr2 = wordTransform[1].strip().replace('"', '')
wordTransform[1].strip().replace('"','')) content = content.replace(replaceStr1, replaceStr2)
return content return content
def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str: def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
"""Uses the tags to replace :emoji: with html image markup """Uses the tags to replace :emoji: with html image markup
""" """
@ -66,14 +67,15 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
if iconName[0].isdigit(): if iconName[0].isdigit():
if '.' in iconName: if '.' in iconName:
iconName = iconName.split('.')[0] iconName = iconName.split('.')[0]
# see https://unicode.org/emoji/charts/full-emoji-list.html # see https://unicode.org/
# emoji/charts/full-emoji-list.html
if '-' not in iconName: if '-' not in iconName:
# a single code # a single code
try: try:
content= \ replaceChar = chr(int("0x" + iconName, 16))
content.replace(tagItem['name'], \ content = content.replace(tagItem['name'],
chr(int("0x"+iconName,16))) replaceChar)
except: except BaseException:
pass pass
else: else:
# sequence of codes # sequence of codes
@ -81,13 +83,13 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
iconCodeSequence = '' iconCodeSequence = ''
for icode in iconCodes: for icode in iconCodes:
try: try:
iconCodeSequence+=chr(int("0x"+icode,16)) iconCodeSequence += chr(int("0x" +
except: icode, 16))
except BaseException:
iconCodeSequence = '' iconCodeSequence = ''
break break
if iconCodeSequence: if iconCodeSequence:
content= \ content = content.replace(tagItem['name'],
content.replace(tagItem['name'], \
iconCodeSequence) iconCodeSequence)
htmlClass = 'emoji' htmlClass = 'emoji'
@ -95,8 +97,7 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
htmlClass = 'emojiheader' htmlClass = 'emojiheader'
if messageType == 'profile': if messageType == 'profile':
htmlClass = 'emojiprofile' htmlClass = 'emojiprofile'
emojiHtml= \ emojiHtml = "<img src=\"" + tagItem['icon']['url'] + "\" alt=\"" + \
"<img src=\""+tagItem['icon']['url']+"\" alt=\""+ \
tagItem['name'].replace(':', '') + \ tagItem['name'].replace(':', '') + \
"\" align=\"middle\" class=\"" + htmlClass + "\"/>" "\" align=\"middle\" class=\"" + htmlClass + "\"/>"
content = content.replace(tagItem['name'], emojiHtml) content = content.replace(tagItem['name'], emojiHtml)
@ -121,6 +122,7 @@ def addMusicTag(content: str,tag: str) -> str:
return content return content
return ':music: ' + content + ' ' + tag + ' ' return ':music: ' + content + ' ' + tag + ' '
def addWebLinks(content: str) -> str: def addWebLinks(content: str) -> str:
"""Adds markup for web links """Adds markup for web links
""" """
@ -137,7 +139,8 @@ def addWebLinks(content: str) -> str:
w.startswith('dat://'): w.startswith('dat://'):
if w.endswith('.') or w.endswith(';'): if w.endswith('.') or w.endswith(';'):
w = w[:-1] w = w[:-1]
markup='<a href="'+w+'" rel="nofollow noopener" target="_blank">' markup = '<a href="' + w + \
'" rel="nofollow noopener" target="_blank">'
if w.startswith('https://'): if w.startswith('https://'):
markup += '<span class="invisible">https://</span>' markup += '<span class="invisible">https://</span>'
elif w.startswith('http://'): elif w.startswith('http://'):
@ -146,15 +149,13 @@ def addWebLinks(content: str) -> str:
markup += '<span class="invisible">i2p://</span>' markup += '<span class="invisible">i2p://</span>'
elif w.startswith('dat://'): elif w.startswith('dat://'):
markup += '<span class="invisible">dat://</span>' markup += '<span class="invisible">dat://</span>'
linkText= \ linkText = w.replace('https://', '').replace('http://', '')
w.replace('https://','').replace('http://','').replace('dat://','').replace('i2p://','') linkText = linkText.replace('dat://', '').replace('i2p://', '')
# prevent links from becoming too long # prevent links from becoming too long
if len(linkText) > maxLinkLength: if len(linkText) > maxLinkLength:
markup+= \ markup += '<span class="ellipsis">' + \
'<span class="ellipsis">'+ \
linkText[:maxLinkLength] + '</span>' linkText[:maxLinkLength] + '</span>'
markup+= \ markup += '<span class="invisible">' + \
'<span class="invisible">'+ \
linkText[maxLinkLength:] + '</span></a>' linkText[maxLinkLength:] + '</span></a>'
else: else:
markup += '<span class="ellipsis">' + linkText + '</span></a>' markup += '<span class="ellipsis">' + linkText + '</span></a>'
@ -164,16 +165,19 @@ def addWebLinks(content: str) -> str:
content = content.replace(' --linebreak-- ', '<br>') content = content.replace(' --linebreak-- ', '<br>')
return content return content
def validHashTag(hashtag: str) -> bool: def validHashTag(hashtag: str) -> bool:
"""Returns true if the give hashtag contains valid characters """Returns true if the give hashtag contains valid characters
""" """
validChars= \ validChars = set('0123456789' +
set('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') 'abcdefghijklmnopqrstuvwxyz' +
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
if set(hashtag).issubset(validChars): if set(hashtag).issubset(validChars):
return True return True
return False return False
def addHashTags(wordStr: str,httpPrefix: str,domain: str, \
def addHashTags(wordStr: str, httpPrefix: str, domain: str,
replaceHashTags: {}, postHashtags: {}) -> bool: replaceHashTags: {}, postHashtags: {}) -> bool:
"""Detects hashtags and adds them to the replacements dict """Detects hashtags and adds them to the replacements dict
Also updates the hashtags list to be added to the post Also updates the hashtags list to be added to the post
@ -189,12 +193,12 @@ def addHashTags(wordStr: str,httpPrefix: str,domain: str, \
'name': '#'+hashtag, 'name': '#'+hashtag,
'type': 'Hashtag' 'type': 'Hashtag'
} }
replaceHashTags[wordStr]= \ replaceHashTags[wordStr] = "<a href=\"" + hashtagUrl + \
"<a href=\""+hashtagUrl+ \
"\" class=\"mention hashtag\" rel=\"tag\">#<span>" + \ "\" class=\"mention hashtag\" rel=\"tag\">#<span>" + \
hashtag + "</span></a>" hashtag + "</span></a>"
return True return True
def loadEmojiDict(emojiDataFilename: str, emojiDict: {}) -> None: def loadEmojiDict(emojiDataFilename: str, emojiDict: {}) -> None:
"""Creates an emoji dictionary based on emoji/emoji-data.txt """Creates an emoji dictionary based on emoji/emoji-data.txt
""" """
@ -215,15 +219,16 @@ def loadEmojiDict(emojiDataFilename: str,emojiDict: {}) -> None:
continue continue
if '..' in emojiUnicode: if '..' in emojiUnicode:
emojiUnicode = emojiUnicode.split('..')[0] emojiUnicode = emojiUnicode.split('..')[0]
emojiName= \ emojiName = line.split(')', 1)[1].strip().replace('\n', '')
line.split(')',1)[1].strip().replace('\n','').replace(' ','').replace('-','') emojiName = emojiName.replace(' ', '').replace('-', '')
if '..' in emojiName: if '..' in emojiName:
emojiName = emojiName.split('..')[0] emojiName = emojiName.split('..')[0]
emojiDict[emojiName.lower()] = emojiUnicode emojiDict[emojiName.lower()] = emojiUnicode
def addEmoji(baseDir: str,wordStr: str, \
httpPrefix: str,domain: str, \ def addEmoji(baseDir: str, wordStr: str,
replaceEmoji: {},postTags: {}, \ httpPrefix: str, domain: str,
replaceEmoji: {}, postTags: {},
emojiDict: {}) -> bool: emojiDict: {}) -> bool:
"""Detects Emoji and adds them to the replacements dict """Detects Emoji and adds them to the replacements dict
Also updates the tags list to be added to the post Also updates the tags list to be added to the post
@ -247,7 +252,8 @@ def addEmoji(baseDir: str,wordStr: str, \
emojiFilename = baseDir + '/emoji/' + emojiDict[emoji] + '.png' emojiFilename = baseDir + '/emoji/' + emojiDict[emoji] + '.png'
if not os.path.isfile(emojiFilename): if not os.path.isfile(emojiFilename):
return False return False
emojiUrl=httpPrefix+"://"+domain+"/emoji/"+emojiDict[emoji]+'.png' emojiUrl = httpPrefix + "://" + domain + \
"/emoji/" + emojiDict[emoji] + '.png'
postTags[emoji] = { postTags[emoji] = {
'icon': { 'icon': {
'mediaType': 'image/png', 'mediaType': 'image/png',
@ -261,7 +267,8 @@ def addEmoji(baseDir: str,wordStr: str, \
} }
return True return True
def addMention(wordStr: str,httpPrefix: str,following: str, \
def addMention(wordStr: str, httpPrefix: str, following: str,
replaceMentions: {}, recipients: [], tags: {}) -> bool: replaceMentions: {}, recipients: [], tags: {}) -> bool:
"""Detects mentions and adds them to the replacements dict and """Detects mentions and adds them to the replacements dict and
recipients list recipients list
@ -275,8 +282,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
for follow in following: for follow in following:
if follow.startswith(possibleNickname + '@'): if follow.startswith(possibleNickname + '@'):
replaceDomain = follow.replace('\n', '').split('@')[1] replaceDomain = follow.replace('\n', '').split('@')[1]
recipientActor= \ recipientActor = httpPrefix + "://" + \
httpPrefix+"://"+replaceDomain+"/users/"+possibleNickname replaceDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr] = { tags[wordStr] = {
@ -305,8 +312,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
for follow in following: for follow in following:
if follow.replace('\n', '') != possibleHandle: if follow.replace('\n', '') != possibleHandle:
continue continue
recipientActor= \ recipientActor = httpPrefix + "://" + \
httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname possibleDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr] = { tags[wordStr] = {
@ -323,7 +330,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
# @nick@domain # @nick@domain
if not (possibleDomain == 'localhost' or '.' in possibleDomain): if not (possibleDomain == 'localhost' or '.' in possibleDomain):
return False return False
recipientActor=httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname recipientActor = httpPrefix + "://" + \
possibleDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr] = { tags[wordStr] = {
@ -338,17 +346,19 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
"</span></a></span>" "</span></a></span>"
return True return True
def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
def removeLongWords(content: str, maxWordLength: int,
longWordsList: []) -> str:
"""Breaks up long words so that on mobile screens this doesn't """Breaks up long words so that on mobile screens this doesn't
disrupt the layout disrupt the layout
""" """
if ' ' not in content: if ' ' not in content:
# handle a single very long string with no spaces # handle a single very long string with no spaces
contentStr=content.replace('<p>','').replace('<\p>','') contentStr = content.replace('<p>', '').replace(r'<\p>', '')
if '://' not in contentStr: if '://' not in contentStr:
if len(contentStr) > maxWordLength: if len(contentStr) > maxWordLength:
if '<p>' in content: if '<p>' in content:
content='<p>'+contentStr[:maxWordLength]+'<\p>' content = '<p>' + contentStr[:maxWordLength] + r'<\p>'
else: else:
content = content[:maxWordLength] content = content[:maxWordLength]
return content return content
@ -387,27 +397,25 @@ def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
continue continue
if '<' in wordStr: if '<' in wordStr:
replaceWord = wordStr.split('<', 1)[0] replaceWord = wordStr.split('<', 1)[0]
content= \ content = content.replace(wordStr, replaceWord)
content.replace(wordStr,replaceWord)
wordStr = replaceWord wordStr = replaceWord
if '/' in wordStr: if '/' in wordStr:
continue continue
if len(wordStr[maxWordLength:]) < maxWordLength: if len(wordStr[maxWordLength:]) < maxWordLength:
content= \ content = content.replace(wordStr,
content.replace(wordStr, \ wordStr[:maxWordLength] + '\n' +
wordStr[:maxWordLength]+'\n'+ \
wordStr[maxWordLength:]) wordStr[maxWordLength:])
else: else:
content= \ content = content.replace(wordStr,
content.replace(wordStr, \
wordStr[:maxWordLength]) wordStr[:maxWordLength])
if content.startswith('<p>'): if content.startswith('<p>'):
if not content.endswith('</p>'): if not content.endswith('</p>'):
content = content.strip()+'</p>' content = content.strip()+'</p>'
return content return content
def addHtmlTags(baseDir: str,httpPrefix: str, \
nickname: str,domain: str,content: str, \ def addHtmlTags(baseDir: str, httpPrefix: str,
nickname: str, domain: str, content: str,
recipients: [], hashtags: {}, isJsonContent=False) -> str: recipients: [], hashtags: {}, isJsonContent=False) -> str:
""" Replaces plaintext mentions such as @nick@domain into html """ Replaces plaintext mentions such as @nick@domain into html
by matching against known following accounts by matching against known following accounts
@ -420,7 +428,6 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
words = content.replace(',', ' ').replace(';', ' ').split(' ') words = content.replace(',', ' ').replace(';', ' ').split(' ')
# remove . for words which are not mentions # remove . for words which are not mentions
wordCtr=0
newWords = [] newWords = []
for wordIndex in range(0, len(words)): for wordIndex in range(0, len(words)):
wordStr = words[wordIndex] wordStr = words[wordIndex]
@ -439,8 +446,8 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
originalDomain = domain originalDomain = domain
if ':' in domain: if ':' in domain:
domain = domain.split(':')[0] domain = domain.split(':')[0]
followingFilename= \ followingFilename = baseDir + '/accounts/' + \
baseDir+'/accounts/'+nickname+'@'+domain+'/following.txt' nickname + '@' + domain + '/following.txt'
# read the following list so that we can detect just @nick # read the following list so that we can detect just @nick
# in addition to @nick@domain # in addition to @nick@domain
@ -459,28 +466,28 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
longWordsList.append(wordStr) longWordsList.append(wordStr)
firstChar = wordStr[0] firstChar = wordStr[0]
if firstChar == '@': if firstChar == '@':
if addMention(wordStr,httpPrefix,following, \ if addMention(wordStr, httpPrefix, following,
replaceMentions, recipients, hashtags): replaceMentions, recipients, hashtags):
continue continue
elif firstChar == '#': elif firstChar == '#':
if addHashTags(wordStr,httpPrefix,originalDomain, \ if addHashTags(wordStr, httpPrefix, originalDomain,
replaceHashTags, hashtags): replaceHashTags, hashtags):
continue continue
elif ':' in wordStr: elif ':' in wordStr:
#print('TAG: emoji located - '+wordStr)
wordStr2 = wordStr.split(':')[1] wordStr2 = wordStr.split(':')[1]
# print('TAG: emoji located - '+wordStr)
if not emojiDict: if not emojiDict:
# emoji.json is generated so that it can be customized and # emoji.json is generated so that it can be customized and
# the changes will be retained even if default_emoji.json # the changes will be retained even if default_emoji.json
# is subsequently updated # is subsequently updated
if not os.path.isfile(baseDir + '/emoji/emoji.json'): if not os.path.isfile(baseDir + '/emoji/emoji.json'):
copyfile(baseDir+'/emoji/default_emoji.json', \ copyfile(baseDir + '/emoji/default_emoji.json',
baseDir + '/emoji/emoji.json') baseDir + '/emoji/emoji.json')
emojiDict = loadJson(baseDir + '/emoji/emoji.json') emojiDict = loadJson(baseDir + '/emoji/emoji.json')
# print('TAG: looking up emoji for :'+wordStr2+':') # print('TAG: looking up emoji for :'+wordStr2+':')
addEmoji(baseDir,':'+wordStr2+':',httpPrefix, \ addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix,
originalDomain,replaceEmoji,hashtags, \ originalDomain, replaceEmoji, hashtags,
emojiDict) emojiDict)
# replace words with their html versions # replace words with their html versions
@ -498,7 +505,8 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
content = content.replace(' --linebreak-- ', '</p><p>') content = content.replace(' --linebreak-- ', '</p><p>')
return '<p>' + content + '</p>' return '<p>' + content + '</p>'
def getMentionsFromHtml(htmlText: str, \
def getMentionsFromHtml(htmlText: str,
matchStr="<span class=\"h-card\"><a href=\"") -> []: matchStr="<span class=\"h-card\"><a href=\"") -> []:
"""Extracts mentioned actors from the given html content string """Extracts mentioned actors from the given html content string
""" """
@ -517,13 +525,13 @@ def getMentionsFromHtml(htmlText: str, \
mentions.append(actorStr) mentions.append(actorStr)
return mentions return mentions
def extractMediaInFormPOST(postBytes, boundary, name: str): def extractMediaInFormPOST(postBytes, boundary, name: str):
"""Extracts the binary encoding for image/video/audio within a http """Extracts the binary encoding for image/video/audio within a http
form POST form POST
Returns the media bytes and the remaining bytes Returns the media bytes and the remaining bytes
""" """
imageStartBoundary= \ imageStartBoundary = b'Content-Disposition: form-data; name="' + \
b'Content-Disposition: form-data; name="'+ \
name.encode('utf8', 'ignore') + b'";' name.encode('utf8', 'ignore') + b'";'
imageStartLocation = postBytes.find(imageStartBoundary) imageStartLocation = postBytes.find(imageStartBoundary)
if imageStartLocation == -1: if imageStartLocation == -1:
@ -548,7 +556,8 @@ def extractMediaInFormPOST(postBytes,boundary,name: str):
# return the media and the before+after bytes # return the media and the before+after bytes
return mediaBytes, postBytes[:imageStartLocation] + remainder return mediaBytes, postBytes[:imageStartLocation] + remainder
def saveMediaInFormPOST(mediaBytes,debug: bool, \
def saveMediaInFormPOST(mediaBytes, debug: bool,
filenameBase=None) -> (str, str): filenameBase=None) -> (str, str):
"""Saves the given media bytes extracted from http form POST """Saves the given media bytes extracted from http form POST
Returns the filename and attachment type Returns the filename and attachment type
@ -607,7 +616,9 @@ def saveMediaInFormPOST(mediaBytes,debug: bool, \
if ex == detectedExtension: if ex == detectedExtension:
continue continue
possibleOtherFormat = \ possibleOtherFormat = \
filename.replace('.temp','').replace('.'+detectedExtension,'.'+ex) filename.replace('.temp', '').replace('.' +
detectedExtension, '.' +
ex)
if os.path.isfile(possibleOtherFormat): if os.path.isfile(possibleOtherFormat):
os.remove(possibleOtherFormat) os.remove(possibleOtherFormat)
@ -617,15 +628,17 @@ def saveMediaInFormPOST(mediaBytes,debug: bool, \
return filename, attachmentMediaType return filename, attachmentMediaType
def extractTextFieldsInPOST(postBytes, boundary, debug: bool) -> {}: def extractTextFieldsInPOST(postBytes, boundary, debug: bool) -> {}:
"""Returns a dictionary containing the text fields of a http form POST """Returns a dictionary containing the text fields of a http form POST
The boundary argument comes from the http header The boundary argument comes from the http header
""" """
msg = email.parser.BytesParser().parsebytes(postBytes) msg = email.parser.BytesParser().parsebytes(postBytes)
if debug: if debug:
print('DEBUG: POST arriving '+ \ print('DEBUG: POST arriving ' +
msg.get_payload(decode=True).decode('utf-8')) msg.get_payload(decode=True).decode('utf-8'))
messageFields=msg.get_payload(decode=True).decode('utf-8').split(boundary) messageFields = msg.get_payload(decode=True)
messageFields = messageFields.decode('utf-8').split(boundary)
fields = {} fields = {}
# examine each section of the POST, separated by the boundary # examine each section of the POST, separated by the boundary
for f in messageFields: for f in messageFields: