forked from indymedia/epicyon
flake8 style
parent
a70793b616
commit
dbcfbd1a8c
151
content.py
151
content.py
|
|
@ -7,17 +7,17 @@ __email__="bob@freedombone.net"
|
|||
__status__ = "Production"
|
||||
|
||||
import os
|
||||
import time
|
||||
import email.parser
|
||||
from shutil import copyfile
|
||||
from utils import loadJson
|
||||
from utils import fileLastModified
|
||||
|
||||
|
||||
def switchWords(baseDir: str, nickname: str, domain: str, content: str) -> str:
|
||||
"""Performs word replacements. eg. Trump -> The Orange Menace
|
||||
"""
|
||||
switchWordsFilename= \
|
||||
baseDir+'/accounts/'+nickname+'@'+domain+'/replacewords.txt'
|
||||
switchWordsFilename = baseDir + '/accounts/' + \
|
||||
nickname + '@' + domain + '/replacewords.txt'
|
||||
if not os.path.isfile(switchWordsFilename):
|
||||
return content
|
||||
with open(switchWordsFilename, 'r') as fp:
|
||||
|
|
@ -37,11 +37,12 @@ def switchWords(baseDir: str,nickname: str,domain: str,content: str) -> str:
|
|||
if not wordTransform:
|
||||
continue
|
||||
if len(wordTransform) == 2:
|
||||
content= \
|
||||
content.replace(wordTransform[0].strip().replace('"',''), \
|
||||
wordTransform[1].strip().replace('"',''))
|
||||
replaceStr1 = wordTransform[0].strip().replace('"', '')
|
||||
replaceStr2 = wordTransform[1].strip().replace('"', '')
|
||||
content = content.replace(replaceStr1, replaceStr2)
|
||||
return content
|
||||
|
||||
|
||||
def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
|
||||
"""Uses the tags to replace :emoji: with html image markup
|
||||
"""
|
||||
|
|
@ -66,14 +67,15 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
|
|||
if iconName[0].isdigit():
|
||||
if '.' in iconName:
|
||||
iconName = iconName.split('.')[0]
|
||||
# see https://unicode.org/emoji/charts/full-emoji-list.html
|
||||
# see https://unicode.org/
|
||||
# emoji/charts/full-emoji-list.html
|
||||
if '-' not in iconName:
|
||||
# a single code
|
||||
try:
|
||||
content= \
|
||||
content.replace(tagItem['name'], \
|
||||
chr(int("0x"+iconName,16)))
|
||||
except:
|
||||
replaceChar = chr(int("0x" + iconName, 16))
|
||||
content = content.replace(tagItem['name'],
|
||||
replaceChar)
|
||||
except BaseException:
|
||||
pass
|
||||
else:
|
||||
# sequence of codes
|
||||
|
|
@ -81,13 +83,13 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
|
|||
iconCodeSequence = ''
|
||||
for icode in iconCodes:
|
||||
try:
|
||||
iconCodeSequence+=chr(int("0x"+icode,16))
|
||||
except:
|
||||
iconCodeSequence += chr(int("0x" +
|
||||
icode, 16))
|
||||
except BaseException:
|
||||
iconCodeSequence = ''
|
||||
break
|
||||
if iconCodeSequence:
|
||||
content= \
|
||||
content.replace(tagItem['name'], \
|
||||
content = content.replace(tagItem['name'],
|
||||
iconCodeSequence)
|
||||
|
||||
htmlClass = 'emoji'
|
||||
|
|
@ -95,8 +97,7 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
|
|||
htmlClass = 'emojiheader'
|
||||
if messageType == 'profile':
|
||||
htmlClass = 'emojiprofile'
|
||||
emojiHtml= \
|
||||
"<img src=\""+tagItem['icon']['url']+"\" alt=\""+ \
|
||||
emojiHtml = "<img src=\"" + tagItem['icon']['url'] + "\" alt=\"" + \
|
||||
tagItem['name'].replace(':', '') + \
|
||||
"\" align=\"middle\" class=\"" + htmlClass + "\"/>"
|
||||
content = content.replace(tagItem['name'], emojiHtml)
|
||||
|
|
@ -121,6 +122,7 @@ def addMusicTag(content: str,tag: str) -> str:
|
|||
return content
|
||||
return ':music: ' + content + ' ' + tag + ' '
|
||||
|
||||
|
||||
def addWebLinks(content: str) -> str:
|
||||
"""Adds markup for web links
|
||||
"""
|
||||
|
|
@ -137,7 +139,8 @@ def addWebLinks(content: str) -> str:
|
|||
w.startswith('dat://'):
|
||||
if w.endswith('.') or w.endswith(';'):
|
||||
w = w[:-1]
|
||||
markup='<a href="'+w+'" rel="nofollow noopener" target="_blank">'
|
||||
markup = '<a href="' + w + \
|
||||
'" rel="nofollow noopener" target="_blank">'
|
||||
if w.startswith('https://'):
|
||||
markup += '<span class="invisible">https://</span>'
|
||||
elif w.startswith('http://'):
|
||||
|
|
@ -146,15 +149,13 @@ def addWebLinks(content: str) -> str:
|
|||
markup += '<span class="invisible">i2p://</span>'
|
||||
elif w.startswith('dat://'):
|
||||
markup += '<span class="invisible">dat://</span>'
|
||||
linkText= \
|
||||
w.replace('https://','').replace('http://','').replace('dat://','').replace('i2p://','')
|
||||
linkText = w.replace('https://', '').replace('http://', '')
|
||||
linkText = linkText.replace('dat://', '').replace('i2p://', '')
|
||||
# prevent links from becoming too long
|
||||
if len(linkText) > maxLinkLength:
|
||||
markup+= \
|
||||
'<span class="ellipsis">'+ \
|
||||
markup += '<span class="ellipsis">' + \
|
||||
linkText[:maxLinkLength] + '</span>'
|
||||
markup+= \
|
||||
'<span class="invisible">'+ \
|
||||
markup += '<span class="invisible">' + \
|
||||
linkText[maxLinkLength:] + '</span></a>'
|
||||
else:
|
||||
markup += '<span class="ellipsis">' + linkText + '</span></a>'
|
||||
|
|
@ -164,16 +165,19 @@ def addWebLinks(content: str) -> str:
|
|||
content = content.replace(' --linebreak-- ', '<br>')
|
||||
return content
|
||||
|
||||
|
||||
def validHashTag(hashtag: str) -> bool:
|
||||
"""Returns true if the give hashtag contains valid characters
|
||||
"""
|
||||
validChars= \
|
||||
set('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')
|
||||
validChars = set('0123456789' +
|
||||
'abcdefghijklmnopqrstuvwxyz' +
|
||||
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
|
||||
if set(hashtag).issubset(validChars):
|
||||
return True
|
||||
return False
|
||||
|
||||
def addHashTags(wordStr: str,httpPrefix: str,domain: str, \
|
||||
|
||||
def addHashTags(wordStr: str, httpPrefix: str, domain: str,
|
||||
replaceHashTags: {}, postHashtags: {}) -> bool:
|
||||
"""Detects hashtags and adds them to the replacements dict
|
||||
Also updates the hashtags list to be added to the post
|
||||
|
|
@ -189,12 +193,12 @@ def addHashTags(wordStr: str,httpPrefix: str,domain: str, \
|
|||
'name': '#'+hashtag,
|
||||
'type': 'Hashtag'
|
||||
}
|
||||
replaceHashTags[wordStr]= \
|
||||
"<a href=\""+hashtagUrl+ \
|
||||
replaceHashTags[wordStr] = "<a href=\"" + hashtagUrl + \
|
||||
"\" class=\"mention hashtag\" rel=\"tag\">#<span>" + \
|
||||
hashtag + "</span></a>"
|
||||
return True
|
||||
|
||||
|
||||
def loadEmojiDict(emojiDataFilename: str, emojiDict: {}) -> None:
|
||||
"""Creates an emoji dictionary based on emoji/emoji-data.txt
|
||||
"""
|
||||
|
|
@ -215,15 +219,16 @@ def loadEmojiDict(emojiDataFilename: str,emojiDict: {}) -> None:
|
|||
continue
|
||||
if '..' in emojiUnicode:
|
||||
emojiUnicode = emojiUnicode.split('..')[0]
|
||||
emojiName= \
|
||||
line.split(')',1)[1].strip().replace('\n','').replace(' ','').replace('-','')
|
||||
emojiName = line.split(')', 1)[1].strip().replace('\n', '')
|
||||
emojiName = emojiName.replace(' ', '').replace('-', '')
|
||||
if '..' in emojiName:
|
||||
emojiName = emojiName.split('..')[0]
|
||||
emojiDict[emojiName.lower()] = emojiUnicode
|
||||
|
||||
def addEmoji(baseDir: str,wordStr: str, \
|
||||
httpPrefix: str,domain: str, \
|
||||
replaceEmoji: {},postTags: {}, \
|
||||
|
||||
def addEmoji(baseDir: str, wordStr: str,
|
||||
httpPrefix: str, domain: str,
|
||||
replaceEmoji: {}, postTags: {},
|
||||
emojiDict: {}) -> bool:
|
||||
"""Detects Emoji and adds them to the replacements dict
|
||||
Also updates the tags list to be added to the post
|
||||
|
|
@ -247,7 +252,8 @@ def addEmoji(baseDir: str,wordStr: str, \
|
|||
emojiFilename = baseDir + '/emoji/' + emojiDict[emoji] + '.png'
|
||||
if not os.path.isfile(emojiFilename):
|
||||
return False
|
||||
emojiUrl=httpPrefix+"://"+domain+"/emoji/"+emojiDict[emoji]+'.png'
|
||||
emojiUrl = httpPrefix + "://" + domain + \
|
||||
"/emoji/" + emojiDict[emoji] + '.png'
|
||||
postTags[emoji] = {
|
||||
'icon': {
|
||||
'mediaType': 'image/png',
|
||||
|
|
@ -261,7 +267,8 @@ def addEmoji(baseDir: str,wordStr: str, \
|
|||
}
|
||||
return True
|
||||
|
||||
def addMention(wordStr: str,httpPrefix: str,following: str, \
|
||||
|
||||
def addMention(wordStr: str, httpPrefix: str, following: str,
|
||||
replaceMentions: {}, recipients: [], tags: {}) -> bool:
|
||||
"""Detects mentions and adds them to the replacements dict and
|
||||
recipients list
|
||||
|
|
@ -275,8 +282,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
|
|||
for follow in following:
|
||||
if follow.startswith(possibleNickname + '@'):
|
||||
replaceDomain = follow.replace('\n', '').split('@')[1]
|
||||
recipientActor= \
|
||||
httpPrefix+"://"+replaceDomain+"/users/"+possibleNickname
|
||||
recipientActor = httpPrefix + "://" + \
|
||||
replaceDomain + "/users/" + possibleNickname
|
||||
if recipientActor not in recipients:
|
||||
recipients.append(recipientActor)
|
||||
tags[wordStr] = {
|
||||
|
|
@ -305,8 +312,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
|
|||
for follow in following:
|
||||
if follow.replace('\n', '') != possibleHandle:
|
||||
continue
|
||||
recipientActor= \
|
||||
httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname
|
||||
recipientActor = httpPrefix + "://" + \
|
||||
possibleDomain + "/users/" + possibleNickname
|
||||
if recipientActor not in recipients:
|
||||
recipients.append(recipientActor)
|
||||
tags[wordStr] = {
|
||||
|
|
@ -323,7 +330,8 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
|
|||
# @nick@domain
|
||||
if not (possibleDomain == 'localhost' or '.' in possibleDomain):
|
||||
return False
|
||||
recipientActor=httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname
|
||||
recipientActor = httpPrefix + "://" + \
|
||||
possibleDomain + "/users/" + possibleNickname
|
||||
if recipientActor not in recipients:
|
||||
recipients.append(recipientActor)
|
||||
tags[wordStr] = {
|
||||
|
|
@ -338,17 +346,19 @@ def addMention(wordStr: str,httpPrefix: str,following: str, \
|
|||
"</span></a></span>"
|
||||
return True
|
||||
|
||||
def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
|
||||
|
||||
def removeLongWords(content: str, maxWordLength: int,
|
||||
longWordsList: []) -> str:
|
||||
"""Breaks up long words so that on mobile screens this doesn't
|
||||
disrupt the layout
|
||||
"""
|
||||
if ' ' not in content:
|
||||
# handle a single very long string with no spaces
|
||||
contentStr=content.replace('<p>','').replace('<\p>','')
|
||||
contentStr = content.replace('<p>', '').replace(r'<\p>', '')
|
||||
if '://' not in contentStr:
|
||||
if len(contentStr) > maxWordLength:
|
||||
if '<p>' in content:
|
||||
content='<p>'+contentStr[:maxWordLength]+'<\p>'
|
||||
content = '<p>' + contentStr[:maxWordLength] + r'<\p>'
|
||||
else:
|
||||
content = content[:maxWordLength]
|
||||
return content
|
||||
|
|
@ -387,27 +397,25 @@ def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
|
|||
continue
|
||||
if '<' in wordStr:
|
||||
replaceWord = wordStr.split('<', 1)[0]
|
||||
content= \
|
||||
content.replace(wordStr,replaceWord)
|
||||
content = content.replace(wordStr, replaceWord)
|
||||
wordStr = replaceWord
|
||||
if '/' in wordStr:
|
||||
continue
|
||||
if len(wordStr[maxWordLength:]) < maxWordLength:
|
||||
content= \
|
||||
content.replace(wordStr, \
|
||||
wordStr[:maxWordLength]+'\n'+ \
|
||||
content = content.replace(wordStr,
|
||||
wordStr[:maxWordLength] + '\n' +
|
||||
wordStr[maxWordLength:])
|
||||
else:
|
||||
content= \
|
||||
content.replace(wordStr, \
|
||||
content = content.replace(wordStr,
|
||||
wordStr[:maxWordLength])
|
||||
if content.startswith('<p>'):
|
||||
if not content.endswith('</p>'):
|
||||
content = content.strip()+'</p>'
|
||||
return content
|
||||
|
||||
def addHtmlTags(baseDir: str,httpPrefix: str, \
|
||||
nickname: str,domain: str,content: str, \
|
||||
|
||||
def addHtmlTags(baseDir: str, httpPrefix: str,
|
||||
nickname: str, domain: str, content: str,
|
||||
recipients: [], hashtags: {}, isJsonContent=False) -> str:
|
||||
""" Replaces plaintext mentions such as @nick@domain into html
|
||||
by matching against known following accounts
|
||||
|
|
@ -420,7 +428,6 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
|
|||
words = content.replace(',', ' ').replace(';', ' ').split(' ')
|
||||
|
||||
# remove . for words which are not mentions
|
||||
wordCtr=0
|
||||
newWords = []
|
||||
for wordIndex in range(0, len(words)):
|
||||
wordStr = words[wordIndex]
|
||||
|
|
@ -439,8 +446,8 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
|
|||
originalDomain = domain
|
||||
if ':' in domain:
|
||||
domain = domain.split(':')[0]
|
||||
followingFilename= \
|
||||
baseDir+'/accounts/'+nickname+'@'+domain+'/following.txt'
|
||||
followingFilename = baseDir + '/accounts/' + \
|
||||
nickname + '@' + domain + '/following.txt'
|
||||
|
||||
# read the following list so that we can detect just @nick
|
||||
# in addition to @nick@domain
|
||||
|
|
@ -459,28 +466,28 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
|
|||
longWordsList.append(wordStr)
|
||||
firstChar = wordStr[0]
|
||||
if firstChar == '@':
|
||||
if addMention(wordStr,httpPrefix,following, \
|
||||
if addMention(wordStr, httpPrefix, following,
|
||||
replaceMentions, recipients, hashtags):
|
||||
continue
|
||||
elif firstChar == '#':
|
||||
if addHashTags(wordStr,httpPrefix,originalDomain, \
|
||||
if addHashTags(wordStr, httpPrefix, originalDomain,
|
||||
replaceHashTags, hashtags):
|
||||
continue
|
||||
elif ':' in wordStr:
|
||||
#print('TAG: emoji located - '+wordStr)
|
||||
wordStr2 = wordStr.split(':')[1]
|
||||
# print('TAG: emoji located - '+wordStr)
|
||||
if not emojiDict:
|
||||
# emoji.json is generated so that it can be customized and
|
||||
# the changes will be retained even if default_emoji.json
|
||||
# is subsequently updated
|
||||
if not os.path.isfile(baseDir + '/emoji/emoji.json'):
|
||||
copyfile(baseDir+'/emoji/default_emoji.json', \
|
||||
copyfile(baseDir + '/emoji/default_emoji.json',
|
||||
baseDir + '/emoji/emoji.json')
|
||||
emojiDict = loadJson(baseDir + '/emoji/emoji.json')
|
||||
|
||||
# print('TAG: looking up emoji for :'+wordStr2+':')
|
||||
addEmoji(baseDir,':'+wordStr2+':',httpPrefix, \
|
||||
originalDomain,replaceEmoji,hashtags, \
|
||||
addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix,
|
||||
originalDomain, replaceEmoji, hashtags,
|
||||
emojiDict)
|
||||
|
||||
# replace words with their html versions
|
||||
|
|
@ -498,7 +505,8 @@ def addHtmlTags(baseDir: str,httpPrefix: str, \
|
|||
content = content.replace(' --linebreak-- ', '</p><p>')
|
||||
return '<p>' + content + '</p>'
|
||||
|
||||
def getMentionsFromHtml(htmlText: str, \
|
||||
|
||||
def getMentionsFromHtml(htmlText: str,
|
||||
matchStr="<span class=\"h-card\"><a href=\"") -> []:
|
||||
"""Extracts mentioned actors from the given html content string
|
||||
"""
|
||||
|
|
@ -517,13 +525,13 @@ def getMentionsFromHtml(htmlText: str, \
|
|||
mentions.append(actorStr)
|
||||
return mentions
|
||||
|
||||
|
||||
def extractMediaInFormPOST(postBytes, boundary, name: str):
|
||||
"""Extracts the binary encoding for image/video/audio within a http
|
||||
form POST
|
||||
Returns the media bytes and the remaining bytes
|
||||
"""
|
||||
imageStartBoundary= \
|
||||
b'Content-Disposition: form-data; name="'+ \
|
||||
imageStartBoundary = b'Content-Disposition: form-data; name="' + \
|
||||
name.encode('utf8', 'ignore') + b'";'
|
||||
imageStartLocation = postBytes.find(imageStartBoundary)
|
||||
if imageStartLocation == -1:
|
||||
|
|
@ -548,7 +556,8 @@ def extractMediaInFormPOST(postBytes,boundary,name: str):
|
|||
# return the media and the before+after bytes
|
||||
return mediaBytes, postBytes[:imageStartLocation] + remainder
|
||||
|
||||
def saveMediaInFormPOST(mediaBytes,debug: bool, \
|
||||
|
||||
def saveMediaInFormPOST(mediaBytes, debug: bool,
|
||||
filenameBase=None) -> (str, str):
|
||||
"""Saves the given media bytes extracted from http form POST
|
||||
Returns the filename and attachment type
|
||||
|
|
@ -607,7 +616,9 @@ def saveMediaInFormPOST(mediaBytes,debug: bool, \
|
|||
if ex == detectedExtension:
|
||||
continue
|
||||
possibleOtherFormat = \
|
||||
filename.replace('.temp','').replace('.'+detectedExtension,'.'+ex)
|
||||
filename.replace('.temp', '').replace('.' +
|
||||
detectedExtension, '.' +
|
||||
ex)
|
||||
if os.path.isfile(possibleOtherFormat):
|
||||
os.remove(possibleOtherFormat)
|
||||
|
||||
|
|
@ -617,15 +628,17 @@ def saveMediaInFormPOST(mediaBytes,debug: bool, \
|
|||
|
||||
return filename, attachmentMediaType
|
||||
|
||||
|
||||
def extractTextFieldsInPOST(postBytes, boundary, debug: bool) -> {}:
|
||||
"""Returns a dictionary containing the text fields of a http form POST
|
||||
The boundary argument comes from the http header
|
||||
"""
|
||||
msg = email.parser.BytesParser().parsebytes(postBytes)
|
||||
if debug:
|
||||
print('DEBUG: POST arriving '+ \
|
||||
print('DEBUG: POST arriving ' +
|
||||
msg.get_payload(decode=True).decode('utf-8'))
|
||||
messageFields=msg.get_payload(decode=True).decode('utf-8').split(boundary)
|
||||
messageFields = msg.get_payload(decode=True)
|
||||
messageFields = messageFields.decode('utf-8').split(boundary)
|
||||
fields = {}
|
||||
# examine each section of the POST, separated by the boundary
|
||||
for f in messageFields:
|
||||
|
|
|
|||
Loading…
Reference in New Issue