mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

flake8 style

main
Bob Mottram 2020-04-02 09:56:17 +00:00
parent a70793b616
commit dbcfbd1a8c
1 changed files with 302 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

581
content.py
View File

@ -1,54 +1,55 @@
__filename__="content.py" __filename__ = "content.py"
__author__="Bob Mottram" __author__ = "Bob Mottram"
__license__="AGPL3+" __license__ = "AGPL3+"
__version__="1.1.0" __version__ = "1.1.0"
__maintainer__="Bob Mottram" __maintainer__ = "Bob Mottram"
__email__="bob@freedombone.net" __email__ = "bob@freedombone.net"
__status__="Production" __status__ = "Production"
import os import os
import time
import email.parser import email.parser
from shutil import copyfile from shutil import copyfile
from utils import loadJson from utils import loadJson
from utils import fileLastModified from utils import fileLastModified
def switchWords(baseDir: str,nickname: str,domain: str,content: str) -> str:
def switchWords(baseDir: str, nickname: str, domain: str, content: str) -> str:
"""Performs word replacements. eg. Trump -> The Orange Menace """Performs word replacements. eg. Trump -> The Orange Menace
""" """
switchWordsFilename= \ switchWordsFilename = baseDir + '/accounts/' + \
baseDir+'/accounts/'+nickname+'@'+domain+'/replacewords.txt' nickname + '@' + domain + '/replacewords.txt'
if not os.path.isfile(switchWordsFilename): if not os.path.isfile(switchWordsFilename):
return content return content
with open(switchWordsFilename, 'r') as fp: with open(switchWordsFilename, 'r') as fp:
for line in fp: for line in fp:
replaceStr=line.replace('\n','') replaceStr = line.replace('\n', '')
wordTransform=None wordTransform = None
if '->' in replaceStr: if '->' in replaceStr:
wordTransform=replaceStr.split('->') wordTransform = replaceStr.split('->')
elif ':' in replaceStr: elif ':' in replaceStr:
wordTransform=replaceStr.split(':') wordTransform = replaceStr.split(':')
elif ',' in replaceStr: elif ',' in replaceStr:
wordTransform=replaceStr.split(',') wordTransform = replaceStr.split(',')
elif ';' in replaceStr: elif ';' in replaceStr:
wordTransform=replaceStr.split(';') wordTransform = replaceStr.split(';')
elif '-' in replaceStr: elif '-' in replaceStr:
wordTransform=replaceStr.split('-') wordTransform = replaceStr.split('-')
if not wordTransform: if not wordTransform:
continue continue
if len(wordTransform)==2: if len(wordTransform) == 2:
content= \ replaceStr1 = wordTransform[0].strip().replace('"', '')
content.replace(wordTransform[0].strip().replace('"',''), \ replaceStr2 = wordTransform[1].strip().replace('"', '')
wordTransform[1].strip().replace('"','')) content = content.replace(replaceStr1, replaceStr2)
return content return content
def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
def replaceEmojiFromTags(content: str, tag: [], messageType: str) -> str:
"""Uses the tags to replace :emoji: with html image markup """Uses the tags to replace :emoji: with html image markup
""" """
for tagItem in tag: for tagItem in tag:
if not tagItem.get('type'): if not tagItem.get('type'):
continue continue
if tagItem['type']!='Emoji': if tagItem['type'] != 'Emoji':
continue continue
if not tagItem.get('name'): if not tagItem.get('name'):
continue continue
@ -60,66 +61,67 @@ def replaceEmojiFromTags(content: str,tag: [],messageType: str) -> str:
continue continue
if tagItem['name'] not in content: if tagItem['name'] not in content:
continue continue
iconName=tagItem['icon']['url'].split('/')[-1] iconName = tagItem['icon']['url'].split('/')[-1]
if iconName: if iconName:
if len(iconName)>1: if len(iconName) > 1:
if iconName[0].isdigit(): if iconName[0].isdigit():
if '.' in iconName: if '.' in iconName:
iconName=iconName.split('.')[0] iconName = iconName.split('.')[0]
# see https://unicode.org/emoji/charts/full-emoji-list.html # see https://unicode.org/
# emoji/charts/full-emoji-list.html
if '-' not in iconName: if '-' not in iconName:
# a single code # a single code
try: try:
content= \ replaceChar = chr(int("0x" + iconName, 16))
content.replace(tagItem['name'], \ content = content.replace(tagItem['name'],
chr(int("0x"+iconName,16))) replaceChar)
except: except BaseException:
pass pass
else: else:
# sequence of codes # sequence of codes
iconCodes=iconName.split('-') iconCodes = iconName.split('-')
iconCodeSequence='' iconCodeSequence = ''
for icode in iconCodes: for icode in iconCodes:
try: try:
iconCodeSequence+=chr(int("0x"+icode,16)) iconCodeSequence += chr(int("0x" +
except: icode, 16))
iconCodeSequence='' except BaseException:
iconCodeSequence = ''
break break
if iconCodeSequence: if iconCodeSequence:
content= \ content = content.replace(tagItem['name'],
content.replace(tagItem['name'], \
iconCodeSequence) iconCodeSequence)
htmlClass='emoji' htmlClass = 'emoji'
if messageType=='post header': if messageType == 'post header':
htmlClass='emojiheader' htmlClass = 'emojiheader'
if messageType=='profile': if messageType == 'profile':
htmlClass='emojiprofile' htmlClass = 'emojiprofile'
emojiHtml= \ emojiHtml = "<img src=\"" + tagItem['icon']['url'] + "\" alt=\"" + \
"<img src=\""+tagItem['icon']['url']+"\" alt=\""+ \ tagItem['name'].replace(':', '') + \
tagItem['name'].replace(':','')+ \ "\" align=\"middle\" class=\"" + htmlClass + "\"/>"
"\" align=\"middle\" class=\""+htmlClass+"\"/>" content = content.replace(tagItem['name'], emojiHtml)
content=content.replace(tagItem['name'],emojiHtml)
return content return content
def addMusicTag(content: str,tag: str) -> str: def addMusicTag(content: str, tag: str) -> str:
"""If a music link is found then ensure that the post is """If a music link is found then ensure that the post is
tagged appropriately tagged appropriately
""" """
if '#' not in tag: if '#' not in tag:
tag='#'+tag tag = '#'+tag
if tag in content: if tag in content:
return content return content
musicSites=['soundcloud.com','bandcamp.com'] musicSites = ['soundcloud.com', 'bandcamp.com']
musicSiteFound=False musicSiteFound = False
for site in musicSites: for site in musicSites:
if site+'/' in content: if site+'/' in content:
musicSiteFound=True musicSiteFound = True
break break
if not musicSiteFound: if not musicSiteFound:
return content return content
return ':music: '+content+' '+tag+' ' return ':music: ' + content + ' ' + tag + ' '
def addWebLinks(content: str) -> str: def addWebLinks(content: str) -> str:
"""Adds markup for web links """Adds markup for web links
@ -127,82 +129,84 @@ def addWebLinks(content: str) -> str:
if not ('https://' in content or 'http://' in content): if not ('https://' in content or 'http://' in content):
return content return content
maxLinkLength=40 maxLinkLength = 40
words=content.replace('\n',' --linebreak-- ').split(' ') words = content.replace('\n', ' --linebreak-- ').split(' ')
replaceDict={} replaceDict = {}
for w in words: for w in words:
if w.startswith('https://') or \ if w.startswith('https://') or \
w.startswith('http://') or \ w.startswith('http://') or \
w.startswith('i2p://') or \ w.startswith('i2p://') or \
w.startswith('dat://'): w.startswith('dat://'):
if w.endswith('.') or w.endswith(';'): if w.endswith('.') or w.endswith(';'):
w=w[:-1] w = w[:-1]
markup='<a href="'+w+'" rel="nofollow noopener" target="_blank">' markup = '<a href="' + w + \
'" rel="nofollow noopener" target="_blank">'
if w.startswith('https://'): if w.startswith('https://'):
markup+='<span class="invisible">https://</span>' markup += '<span class="invisible">https://</span>'
elif w.startswith('http://'): elif w.startswith('http://'):
markup+='<span class="invisible">http://</span>' markup += '<span class="invisible">http://</span>'
elif w.startswith('i2p://'): elif w.startswith('i2p://'):
markup+='<span class="invisible">i2p://</span>' markup += '<span class="invisible">i2p://</span>'
elif w.startswith('dat://'): elif w.startswith('dat://'):
markup+='<span class="invisible">dat://</span>' markup += '<span class="invisible">dat://</span>'
linkText= \ linkText = w.replace('https://', '').replace('http://', '')
w.replace('https://','').replace('http://','').replace('dat://','').replace('i2p://','') linkText = linkText.replace('dat://', '').replace('i2p://', '')
# prevent links from becoming too long # prevent links from becoming too long
if len(linkText)>maxLinkLength: if len(linkText) > maxLinkLength:
markup+= \ markup += '<span class="ellipsis">' + \
'<span class="ellipsis">'+ \ linkText[:maxLinkLength] + '</span>'
linkText[:maxLinkLength]+'</span>' markup += '<span class="invisible">' + \
markup+= \ linkText[maxLinkLength:] + '</span></a>'
'<span class="invisible">'+ \
linkText[maxLinkLength:]+'</span></a>'
else: else:
markup+='<span class="ellipsis">'+linkText+'</span></a>' markup += '<span class="ellipsis">' + linkText + '</span></a>'
replaceDict[w]=markup replaceDict[w] = markup
for url,markup in replaceDict.items(): for url, markup in replaceDict.items():
content=content.replace(url,markup) content = content.replace(url, markup)
content=content.replace(' --linebreak-- ','<br>') content = content.replace(' --linebreak-- ', '<br>')
return content return content
def validHashTag(hashtag: str) -> bool: def validHashTag(hashtag: str) -> bool:
"""Returns true if the give hashtag contains valid characters """Returns true if the give hashtag contains valid characters
""" """
validChars= \ validChars = set('0123456789' +
set('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') 'abcdefghijklmnopqrstuvwxyz' +
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
if set(hashtag).issubset(validChars): if set(hashtag).issubset(validChars):
return True return True
return False return False
def addHashTags(wordStr: str,httpPrefix: str,domain: str, \
replaceHashTags: {},postHashtags: {}) -> bool: def addHashTags(wordStr: str, httpPrefix: str, domain: str,
replaceHashTags: {}, postHashtags: {}) -> bool:
"""Detects hashtags and adds them to the replacements dict """Detects hashtags and adds them to the replacements dict
Also updates the hashtags list to be added to the post Also updates the hashtags list to be added to the post
""" """
if replaceHashTags.get(wordStr): if replaceHashTags.get(wordStr):
return True return True
hashtag=wordStr[1:] hashtag = wordStr[1:]
if not validHashTag(hashtag): if not validHashTag(hashtag):
return False return False
hashtagUrl=httpPrefix+"://"+domain+"/tags/"+hashtag hashtagUrl = httpPrefix + "://" + domain + "/tags/" + hashtag
postHashtags[hashtag]= { postHashtags[hashtag] = {
'href': hashtagUrl, 'href': hashtagUrl,
'name': '#'+hashtag, 'name': '#'+hashtag,
'type': 'Hashtag' 'type': 'Hashtag'
} }
replaceHashTags[wordStr]= \ replaceHashTags[wordStr] = "<a href=\"" + hashtagUrl + \
"<a href=\""+hashtagUrl+ \ "\" class=\"mention hashtag\" rel=\"tag\">#<span>" + \
"\" class=\"mention hashtag\" rel=\"tag\">#<span>"+ \ hashtag + "</span></a>"
hashtag+"</span></a>"
return True return True
def loadEmojiDict(emojiDataFilename: str,emojiDict: {}) -> None:
def loadEmojiDict(emojiDataFilename: str, emojiDict: {}) -> None:
"""Creates an emoji dictionary based on emoji/emoji-data.txt """Creates an emoji dictionary based on emoji/emoji-data.txt
""" """
if not os.path.isfile(emojiDataFilename): if not os.path.isfile(emojiDataFilename):
return return
with open (emojiDataFilename, "r") as fileHandler: with open(emojiDataFilename, "r") as fileHandler:
for line in fileHandler: for line in fileHandler:
if len(line)<5: if len(line) < 5:
continue continue
if line.startswith('#'): if line.startswith('#'):
continue continue
@ -210,20 +214,21 @@ def loadEmojiDict(emojiDataFilename: str,emojiDict: {}) -> None:
continue continue
if ')' not in line: if ')' not in line:
continue continue
emojiUnicode=line.split(' ')[0] emojiUnicode = line.split(' ')[0]
if len(emojiUnicode)<4: if len(emojiUnicode) < 4:
continue continue
if '..' in emojiUnicode: if '..' in emojiUnicode:
emojiUnicode=emojiUnicode.split('..')[0] emojiUnicode = emojiUnicode.split('..')[0]
emojiName= \ emojiName = line.split(')', 1)[1].strip().replace('\n', '')
line.split(')',1)[1].strip().replace('\n','').replace(' ','').replace('-','') emojiName = emojiName.replace(' ', '').replace('-', '')
if '..' in emojiName: if '..' in emojiName:
emojiName=emojiName.split('..')[0] emojiName = emojiName.split('..')[0]
emojiDict[emojiName.lower()]=emojiUnicode emojiDict[emojiName.lower()] = emojiUnicode
def addEmoji(baseDir: str,wordStr: str, \
httpPrefix: str,domain: str, \ def addEmoji(baseDir: str, wordStr: str,
replaceEmoji: {},postTags: {}, \ httpPrefix: str, domain: str,
replaceEmoji: {}, postTags: {},
emojiDict: {}) -> bool: emojiDict: {}) -> bool:
"""Detects Emoji and adds them to the replacements dict """Detects Emoji and adds them to the replacements dict
Also updates the tags list to be added to the post Also updates the tags list to be added to the post
@ -232,23 +237,24 @@ def addEmoji(baseDir: str,wordStr: str, \
return False return False
if not wordStr.endswith(':'): if not wordStr.endswith(':'):
return False return False
if len(wordStr)<3: if len(wordStr) < 3:
return False return False
if replaceEmoji.get(wordStr): if replaceEmoji.get(wordStr):
return True return True
# remove leading and trailing : characters # remove leading and trailing : characters
emoji=wordStr[1:] emoji = wordStr[1:]
emoji=emoji[:-1] emoji = emoji[:-1]
# is the text of the emoji valid? # is the text of the emoji valid?
if not validHashTag(emoji): if not validHashTag(emoji):
return False return False
if not emojiDict.get(emoji): if not emojiDict.get(emoji):
return False return False
emojiFilename=baseDir+'/emoji/'+emojiDict[emoji]+'.png' emojiFilename = baseDir + '/emoji/' + emojiDict[emoji] + '.png'
if not os.path.isfile(emojiFilename): if not os.path.isfile(emojiFilename):
return False return False
emojiUrl=httpPrefix+"://"+domain+"/emoji/"+emojiDict[emoji]+'.png' emojiUrl = httpPrefix + "://" + domain + \
postTags[emoji]= { "/emoji/" + emojiDict[emoji] + '.png'
postTags[emoji] = {
'icon': { 'icon': {
'mediaType': 'image/png', 'mediaType': 'image/png',
'type': 'Image', 'type': 'Image',
@ -256,114 +262,118 @@ def addEmoji(baseDir: str,wordStr: str, \
}, },
'name': ':'+emoji+':', 'name': ':'+emoji+':',
"updated": fileLastModified(emojiFilename), "updated": fileLastModified(emojiFilename),
"id": emojiUrl.replace('.png',''), "id": emojiUrl.replace('.png', ''),
'type': 'Emoji' 'type': 'Emoji'
} }
return True return True
def addMention(wordStr: str,httpPrefix: str,following: str, \
replaceMentions: {},recipients: [],tags: {}) -> bool: def addMention(wordStr: str, httpPrefix: str, following: str,
replaceMentions: {}, recipients: [], tags: {}) -> bool:
"""Detects mentions and adds them to the replacements dict and """Detects mentions and adds them to the replacements dict and
recipients list recipients list
""" """
possibleHandle=wordStr[1:] possibleHandle = wordStr[1:]
# @nick # @nick
if following and '@' not in possibleHandle: if following and '@' not in possibleHandle:
# fall back to a best effort match against the following list # fall back to a best effort match against the following list
# if no domain was specified. eg. @nick # if no domain was specified. eg. @nick
possibleNickname=possibleHandle possibleNickname = possibleHandle
for follow in following: for follow in following:
if follow.startswith(possibleNickname+'@'): if follow.startswith(possibleNickname + '@'):
replaceDomain=follow.replace('\n','').split('@')[1] replaceDomain = follow.replace('\n', '').split('@')[1]
recipientActor= \ recipientActor = httpPrefix + "://" + \
httpPrefix+"://"+replaceDomain+"/users/"+possibleNickname replaceDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr]={ tags[wordStr] = {
'href': recipientActor, 'href': recipientActor,
'name': wordStr, 'name': wordStr,
'type': 'Mention' 'type': 'Mention'
} }
replaceMentions[wordStr]= \ replaceMentions[wordStr] = \
"<span class=\"h-card\"><a href=\""+httpPrefix+ \ "<span class=\"h-card\"><a href=\"" + httpPrefix + \
"://"+replaceDomain+"/@"+possibleNickname+ \ "://" + replaceDomain + "/@" + possibleNickname + \
"\" class=\"u-url mention\">@<span>"+possibleNickname+ \ "\" class=\"u-url mention\">@<span>" + possibleNickname + \
"</span></a></span>" "</span></a></span>"
return True return True
return False return False
possibleNickname=None possibleNickname = None
possibleDomain=None possibleDomain = None
if '@' not in possibleHandle: if '@' not in possibleHandle:
return False return False
possibleNickname=possibleHandle.split('@')[0] possibleNickname = possibleHandle.split('@')[0]
if not possibleNickname: if not possibleNickname:
return False return False
possibleDomain=possibleHandle.split('@')[1].strip('\n') possibleDomain = possibleHandle.split('@')[1].strip('\n')
if not possibleDomain: if not possibleDomain:
return False return False
if following: if following:
for follow in following: for follow in following:
if follow.replace('\n','')!=possibleHandle: if follow.replace('\n', '') != possibleHandle:
continue continue
recipientActor= \ recipientActor = httpPrefix + "://" + \
httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname possibleDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr]={ tags[wordStr] = {
'href': recipientActor, 'href': recipientActor,
'name': wordStr, 'name': wordStr,
'type': 'Mention' 'type': 'Mention'
} }
replaceMentions[wordStr]= \ replaceMentions[wordStr] = \
"<span class=\"h-card\"><a href=\""+httpPrefix+ \ "<span class=\"h-card\"><a href=\"" + httpPrefix + \
"://"+possibleDomain+"/@"+possibleNickname+ \ "://" + possibleDomain + "/@" + possibleNickname + \
"\" class=\"u-url mention\">@<span>"+possibleNickname+ \ "\" class=\"u-url mention\">@<span>" + possibleNickname + \
"</span></a></span>" "</span></a></span>"
return True return True
# @nick@domain # @nick@domain
if not (possibleDomain=='localhost' or '.' in possibleDomain): if not (possibleDomain == 'localhost' or '.' in possibleDomain):
return False return False
recipientActor=httpPrefix+"://"+possibleDomain+"/users/"+possibleNickname recipientActor = httpPrefix + "://" + \
possibleDomain + "/users/" + possibleNickname
if recipientActor not in recipients: if recipientActor not in recipients:
recipients.append(recipientActor) recipients.append(recipientActor)
tags[wordStr]={ tags[wordStr] = {
'href': recipientActor, 'href': recipientActor,
'name': wordStr, 'name': wordStr,
'type': 'Mention' 'type': 'Mention'
} }
replaceMentions[wordStr]= \ replaceMentions[wordStr] = \
"<span class=\"h-card\"><a href=\""+httpPrefix+ \ "<span class=\"h-card\"><a href=\"" + httpPrefix + \
"://"+possibleDomain+"/@"+possibleNickname+ \ "://" + possibleDomain + "/@" + possibleNickname + \
"\" class=\"u-url mention\">@<span>"+possibleNickname+ \ "\" class=\"u-url mention\">@<span>" + possibleNickname + \
"</span></a></span>" "</span></a></span>"
return True return True
def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
def removeLongWords(content: str, maxWordLength: int,
longWordsList: []) -> str:
"""Breaks up long words so that on mobile screens this doesn't """Breaks up long words so that on mobile screens this doesn't
disrupt the layout disrupt the layout
""" """
if ' ' not in content: if ' ' not in content:
# handle a single very long string with no spaces # handle a single very long string with no spaces
contentStr=content.replace('<p>','').replace('<\p>','') contentStr = content.replace('<p>', '').replace(r'<\p>', '')
if '://' not in contentStr: if '://' not in contentStr:
if len(contentStr)>maxWordLength: if len(contentStr) > maxWordLength:
if '<p>' in content: if '<p>' in content:
content='<p>'+contentStr[:maxWordLength]+'<\p>' content = '<p>' + contentStr[:maxWordLength] + r'<\p>'
else: else:
content=content[:maxWordLength] content = content[:maxWordLength]
return content return content
words=content.split(' ') words = content.split(' ')
if not longWordsList: if not longWordsList:
longWordsList=[] longWordsList = []
for wordStr in words: for wordStr in words:
if len(wordStr)>maxWordLength: if len(wordStr) > maxWordLength:
if wordStr not in longWordsList: if wordStr not in longWordsList:
longWordsList.append(wordStr) longWordsList.append(wordStr)
for wordStr in longWordsList: for wordStr in longWordsList:
if wordStr.startswith('<'): if wordStr.startswith('<'):
continue continue
if len(wordStr)==76: if len(wordStr) == 76:
if wordStr.upper()==wordStr: if wordStr.upper() == wordStr:
# tox address # tox address
continue continue
if '=\"' in wordStr: if '=\"' in wordStr:
@ -386,130 +396,128 @@ def removeLongWords(content: str,maxWordLength: int,longWordsList: []) -> str:
elif 'dat:' in wordStr: elif 'dat:' in wordStr:
continue continue
if '<' in wordStr: if '<' in wordStr:
replaceWord=wordStr.split('<',1)[0] replaceWord = wordStr.split('<', 1)[0]
content= \ content = content.replace(wordStr, replaceWord)
content.replace(wordStr,replaceWord) wordStr = replaceWord
wordStr=replaceWord
if '/' in wordStr: if '/' in wordStr:
continue continue
if len(wordStr[maxWordLength:])<maxWordLength: if len(wordStr[maxWordLength:]) < maxWordLength:
content= \ content = content.replace(wordStr,
content.replace(wordStr, \ wordStr[:maxWordLength] + '\n' +
wordStr[:maxWordLength]+'\n'+ \
wordStr[maxWordLength:]) wordStr[maxWordLength:])
else: else:
content= \ content = content.replace(wordStr,
content.replace(wordStr, \
wordStr[:maxWordLength]) wordStr[:maxWordLength])
if content.startswith('<p>'): if content.startswith('<p>'):
if not content.endswith('</p>'): if not content.endswith('</p>'):
content=content.strip()+'</p>' content = content.strip()+'</p>'
return content return content
def addHtmlTags(baseDir: str,httpPrefix: str, \
nickname: str,domain: str,content: str, \ def addHtmlTags(baseDir: str, httpPrefix: str,
recipients: [],hashtags: {},isJsonContent=False) -> str: nickname: str, domain: str, content: str,
recipients: [], hashtags: {}, isJsonContent=False) -> str:
""" Replaces plaintext mentions such as @nick@domain into html """ Replaces plaintext mentions such as @nick@domain into html
by matching against known following accounts by matching against known following accounts
""" """
if content.startswith('<p>'): if content.startswith('<p>'):
return content return content
maxWordLength=40 maxWordLength = 40
content=content.replace('\n',' --linebreak-- ') content = content.replace('\n', ' --linebreak-- ')
content=addMusicTag(content,'nowplaying') content = addMusicTag(content, 'nowplaying')
words=content.replace(',',' ').replace(';',' ').split(' ') words = content.replace(',', ' ').replace(';', ' ').split(' ')
# remove . for words which are not mentions # remove . for words which are not mentions
wordCtr=0 newWords = []
newWords=[] for wordIndex in range(0, len(words)):
for wordIndex in range(0,len(words)): wordStr = words[wordIndex]
wordStr=words[wordIndex]
if wordStr.endswith('.'): if wordStr.endswith('.'):
if not wordStr.startswith('@'): if not wordStr.startswith('@'):
wordStr=wordStr[:-1] wordStr = wordStr[:-1]
if wordStr.startswith('.'): if wordStr.startswith('.'):
wordStr=wordStr[1:] wordStr = wordStr[1:]
newWords.append(wordStr) newWords.append(wordStr)
words=newWords words = newWords
replaceMentions={} replaceMentions = {}
replaceHashTags={} replaceHashTags = {}
replaceEmoji={} replaceEmoji = {}
emojiDict={} emojiDict = {}
originalDomain=domain originalDomain = domain
if ':' in domain: if ':' in domain:
domain=domain.split(':')[0] domain = domain.split(':')[0]
followingFilename= \ followingFilename = baseDir + '/accounts/' + \
baseDir+'/accounts/'+nickname+'@'+domain+'/following.txt' nickname + '@' + domain + '/following.txt'
# read the following list so that we can detect just @nick # read the following list so that we can detect just @nick
# in addition to @nick@domain # in addition to @nick@domain
following=None following = None
if '@' in words: if '@' in words:
if os.path.isfile(followingFilename): if os.path.isfile(followingFilename):
with open(followingFilename, "r") as f: with open(followingFilename, "r") as f:
following=f.readlines() following = f.readlines()
# extract mentions and tags from words # extract mentions and tags from words
longWordsList=[] longWordsList = []
for wordStr in words: for wordStr in words:
wordLen=len(wordStr) wordLen = len(wordStr)
if wordLen>2: if wordLen > 2:
if wordLen>maxWordLength: if wordLen > maxWordLength:
longWordsList.append(wordStr) longWordsList.append(wordStr)
firstChar=wordStr[0] firstChar = wordStr[0]
if firstChar=='@': if firstChar == '@':
if addMention(wordStr,httpPrefix,following, \ if addMention(wordStr, httpPrefix, following,
replaceMentions,recipients,hashtags): replaceMentions, recipients, hashtags):
continue continue
elif firstChar=='#': elif firstChar == '#':
if addHashTags(wordStr,httpPrefix,originalDomain, \ if addHashTags(wordStr, httpPrefix, originalDomain,
replaceHashTags,hashtags): replaceHashTags, hashtags):
continue continue
elif ':' in wordStr: elif ':' in wordStr:
#print('TAG: emoji located - '+wordStr) wordStr2 = wordStr.split(':')[1]
wordStr2=wordStr.split(':')[1] # print('TAG: emoji located - '+wordStr)
if not emojiDict: if not emojiDict:
# emoji.json is generated so that it can be customized and # emoji.json is generated so that it can be customized and
# the changes will be retained even if default_emoji.json # the changes will be retained even if default_emoji.json
# is subsequently updated # is subsequently updated
if not os.path.isfile(baseDir+'/emoji/emoji.json'): if not os.path.isfile(baseDir + '/emoji/emoji.json'):
copyfile(baseDir+'/emoji/default_emoji.json', \ copyfile(baseDir + '/emoji/default_emoji.json',
baseDir+'/emoji/emoji.json') baseDir + '/emoji/emoji.json')
emojiDict=loadJson(baseDir+'/emoji/emoji.json') emojiDict = loadJson(baseDir + '/emoji/emoji.json')
#print('TAG: looking up emoji for :'+wordStr2+':') # print('TAG: looking up emoji for :'+wordStr2+':')
addEmoji(baseDir,':'+wordStr2+':',httpPrefix, \ addEmoji(baseDir, ':' + wordStr2 + ':', httpPrefix,
originalDomain,replaceEmoji,hashtags, \ originalDomain, replaceEmoji, hashtags,
emojiDict) emojiDict)
# replace words with their html versions # replace words with their html versions
for wordStr,replaceStr in replaceMentions.items(): for wordStr, replaceStr in replaceMentions.items():
content=content.replace(wordStr,replaceStr) content = content.replace(wordStr, replaceStr)
for wordStr,replaceStr in replaceHashTags.items(): for wordStr, replaceStr in replaceHashTags.items():
content=content.replace(wordStr,replaceStr) content = content.replace(wordStr, replaceStr)
if not isJsonContent: if not isJsonContent:
for wordStr,replaceStr in replaceEmoji.items(): for wordStr, replaceStr in replaceEmoji.items():
content=content.replace(wordStr,replaceStr) content = content.replace(wordStr, replaceStr)
content=addWebLinks(content) content = addWebLinks(content)
if longWordsList: if longWordsList:
content=removeLongWords(content,maxWordLength,longWordsList) content = removeLongWords(content, maxWordLength, longWordsList)
content=content.replace(' --linebreak-- ','</p><p>') content = content.replace(' --linebreak-- ', '</p><p>')
return '<p>'+content+'</p>' return '<p>' + content + '</p>'
def getMentionsFromHtml(htmlText: str, \
def getMentionsFromHtml(htmlText: str,
matchStr="<span class=\"h-card\"><a href=\"") -> []: matchStr="<span class=\"h-card\"><a href=\"") -> []:
"""Extracts mentioned actors from the given html content string """Extracts mentioned actors from the given html content string
""" """
mentions=[] mentions = []
if matchStr not in htmlText: if matchStr not in htmlText:
return mentions return mentions
mentionsList=htmlText.split(matchStr) mentionsList = htmlText.split(matchStr)
for mentionStr in mentionsList: for mentionStr in mentionsList:
if '"' not in mentionStr: if '"' not in mentionStr:
continue continue
actorStr=mentionStr.split('"')[0] actorStr = mentionStr.split('"')[0]
if actorStr.startswith('http') or \ if actorStr.startswith('http') or \
actorStr.startswith('i2p') or \ actorStr.startswith('i2p') or \
actorStr.startswith('dat:'): actorStr.startswith('dat:'):
@ -517,54 +525,55 @@ def getMentionsFromHtml(htmlText: str, \
mentions.append(actorStr) mentions.append(actorStr)
return mentions return mentions
def extractMediaInFormPOST(postBytes,boundary,name: str):
def extractMediaInFormPOST(postBytes, boundary, name: str):
"""Extracts the binary encoding for image/video/audio within a http """Extracts the binary encoding for image/video/audio within a http
form POST form POST
Returns the media bytes and the remaining bytes Returns the media bytes and the remaining bytes
""" """
imageStartBoundary= \ imageStartBoundary = b'Content-Disposition: form-data; name="' + \
b'Content-Disposition: form-data; name="'+ \ name.encode('utf8', 'ignore') + b'";'
name.encode('utf8', 'ignore')+b'";' imageStartLocation = postBytes.find(imageStartBoundary)
imageStartLocation=postBytes.find(imageStartBoundary) if imageStartLocation == -1:
if imageStartLocation==-1: return None, postBytes
return None,postBytes
# bytes after the start boundary appears # bytes after the start boundary appears
mediaBytes=postBytes[imageStartLocation:] mediaBytes = postBytes[imageStartLocation:]
# look for the next boundary # look for the next boundary
imageEndBoundary=boundary.encode('utf8', 'ignore') imageEndBoundary = boundary.encode('utf8', 'ignore')
imageEndLocation=mediaBytes.find(imageEndBoundary) imageEndLocation = mediaBytes.find(imageEndBoundary)
if imageEndLocation==-1: if imageEndLocation == -1:
# no ending boundary # no ending boundary
return mediaBytes,postBytes[:imageStartLocation] return mediaBytes, postBytes[:imageStartLocation]
# remaining bytes after the end of the image # remaining bytes after the end of the image
remainder=mediaBytes[imageEndLocation:] remainder = mediaBytes[imageEndLocation:]
# remove bytes after the end boundary # remove bytes after the end boundary
mediaBytes=mediaBytes[:imageEndLocation] mediaBytes = mediaBytes[:imageEndLocation]
# return the media and the before+after bytes # return the media and the before+after bytes
return mediaBytes,postBytes[:imageStartLocation]+remainder return mediaBytes, postBytes[:imageStartLocation] + remainder
def saveMediaInFormPOST(mediaBytes,debug: bool, \
filenameBase=None) -> (str,str): def saveMediaInFormPOST(mediaBytes, debug: bool,
filenameBase=None) -> (str, str):
"""Saves the given media bytes extracted from http form POST """Saves the given media bytes extracted from http form POST
Returns the filename and attachment type Returns the filename and attachment type
""" """
if not mediaBytes: if not mediaBytes:
if debug: if debug:
print('DEBUG: No media found within POST') print('DEBUG: No media found within POST')
return None,None return None, None
mediaLocation=-1 mediaLocation = -1
searchStr='' searchStr = ''
filename=None filename = None
# directly search the binary array for the beginning # directly search the binary array for the beginning
# of an image # of an image
extensionList= { extensionList = {
'png': 'image/png', 'png': 'image/png',
'jpeg': 'image/jpeg', 'jpeg': 'image/jpeg',
'gif': 'image/gif', 'gif': 'image/gif',
@ -574,80 +583,84 @@ def saveMediaInFormPOST(mediaBytes,debug: bool, \
'mp3': 'audio/mpeg', 'mp3': 'audio/mpeg',
'ogg': 'audio/ogg' 'ogg': 'audio/ogg'
} }
detectedExtension=None detectedExtension = None
for extension,contentType in extensionList.items(): for extension, contentType in extensionList.items():
searchStr=b'Content-Type: '+contentType.encode('utf8', 'ignore') searchStr = b'Content-Type: ' + contentType.encode('utf8', 'ignore')
mediaLocation=mediaBytes.find(searchStr) mediaLocation = mediaBytes.find(searchStr)
if mediaLocation>-1: if mediaLocation > -1:
if extension=='jpeg': if extension == 'jpeg':
extension='jpg' extension = 'jpg'
elif extension=='mpeg': elif extension == 'mpeg':
extension='mp3' extension = 'mp3'
filename=filenameBase+'.'+extension filename = filenameBase + '.' + extension
attachmentMediaType= \ attachmentMediaType = \
searchStr.decode().split('/')[0].replace('Content-Type: ','') searchStr.decode().split('/')[0].replace('Content-Type: ', '')
detectedExtension=extension detectedExtension = extension
break break
if not filename: if not filename:
return None,None return None, None
# locate the beginning of the image, after any # locate the beginning of the image, after any
# carriage returns # carriage returns
startPos=mediaLocation+len(searchStr) startPos = mediaLocation + len(searchStr)
for offset in range(1,8): for offset in range(1, 8):
if mediaBytes[startPos+offset]!=10: if mediaBytes[startPos+offset] != 10:
if mediaBytes[startPos+offset]!=13: if mediaBytes[startPos+offset] != 13:
startPos+=offset startPos += offset
break break
# remove any existing image files with a different format # remove any existing image files with a different format
extensionTypes=('png','jpg','jpeg','gif','webp') extensionTypes = ('png', 'jpg', 'jpeg', 'gif', 'webp')
for ex in extensionTypes: for ex in extensionTypes:
if ex==detectedExtension: if ex == detectedExtension:
continue continue
possibleOtherFormat= \ possibleOtherFormat = \
filename.replace('.temp','').replace('.'+detectedExtension,'.'+ex) filename.replace('.temp', '').replace('.' +
detectedExtension, '.' +
ex)
if os.path.isfile(possibleOtherFormat): if os.path.isfile(possibleOtherFormat):
os.remove(possibleOtherFormat) os.remove(possibleOtherFormat)
fd=open(filename, 'wb') fd = open(filename, 'wb')
fd.write(mediaBytes[startPos:]) fd.write(mediaBytes[startPos:])
fd.close() fd.close()
return filename,attachmentMediaType return filename, attachmentMediaType
def extractTextFieldsInPOST(postBytes,boundary,debug: bool) -> {}:
def extractTextFieldsInPOST(postBytes, boundary, debug: bool) -> {}:
"""Returns a dictionary containing the text fields of a http form POST """Returns a dictionary containing the text fields of a http form POST
The boundary argument comes from the http header The boundary argument comes from the http header
""" """
msg=email.parser.BytesParser().parsebytes(postBytes) msg = email.parser.BytesParser().parsebytes(postBytes)
if debug: if debug:
print('DEBUG: POST arriving '+ \ print('DEBUG: POST arriving ' +
msg.get_payload(decode=True).decode('utf-8')) msg.get_payload(decode=True).decode('utf-8'))
messageFields=msg.get_payload(decode=True).decode('utf-8').split(boundary) messageFields = msg.get_payload(decode=True)
fields={} messageFields = messageFields.decode('utf-8').split(boundary)
fields = {}
# examine each section of the POST, separated by the boundary # examine each section of the POST, separated by the boundary
for f in messageFields: for f in messageFields:
if f=='--': if f == '--':
continue continue
if ' name="' not in f: if ' name="' not in f:
continue continue
postStr=f.split(' name="',1)[1] postStr = f.split(' name="', 1)[1]
if '"' not in postStr: if '"' not in postStr:
continue continue
postKey=postStr.split('"',1)[0] postKey = postStr.split('"', 1)[0]
postValueStr=postStr.split('"',1)[1] postValueStr = postStr.split('"', 1)[1]
if ';' in postValueStr: if ';' in postValueStr:
continue continue
if '\r\n' not in postValueStr: if '\r\n' not in postValueStr:
continue continue
postLines=postValueStr.split('\r\n') postLines = postValueStr.split('\r\n')
postValue='' postValue = ''
if len(postLines)>2: if len(postLines) > 2:
for line in range(2,len(postLines)-1): for line in range(2, len(postLines)-1):
if line>2: if line > 2:
postValue+='\n' postValue += '\n'
postValue+=postLines[line] postValue += postLines[line]
fields[postKey]=postValue fields[postKey] = postValue
return fields return fields