mirror of https://gitlab.com/bashrc2/epicyon
Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon
commit
65884aa33a
354
newsdaemon.py
354
newsdaemon.py
|
|
@ -74,6 +74,120 @@ def _removeControlCharacters(content: str) -> str:
|
||||||
return content
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalNot(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" NOT
|
||||||
|
"""
|
||||||
|
if len(tree) != 2:
|
||||||
|
return False
|
||||||
|
if isinstance(tree[1], str):
|
||||||
|
return tree[1] not in hashtags
|
||||||
|
elif isinstance(tree[1], list):
|
||||||
|
return not hashtagRuleResolve(tree[1], hashtags,
|
||||||
|
moderated, content, url)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalContains(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" Contains
|
||||||
|
"""
|
||||||
|
if len(tree) != 2:
|
||||||
|
return False
|
||||||
|
matchStr = None
|
||||||
|
if isinstance(tree[1], str):
|
||||||
|
matchStr = tree[1]
|
||||||
|
elif isinstance(tree[1], list):
|
||||||
|
matchStr = tree[1][0]
|
||||||
|
if matchStr:
|
||||||
|
if matchStr.startswith('"') and matchStr.endswith('"'):
|
||||||
|
matchStr = matchStr[1:]
|
||||||
|
matchStr = matchStr[:len(matchStr) - 1]
|
||||||
|
matchStrLower = matchStr.lower()
|
||||||
|
contentWithoutTags = content.replace('#' + matchStrLower, '')
|
||||||
|
return matchStrLower in contentWithoutTags
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalFrom(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" FROM
|
||||||
|
"""
|
||||||
|
if len(tree) != 2:
|
||||||
|
return False
|
||||||
|
matchStr = None
|
||||||
|
if isinstance(tree[1], str):
|
||||||
|
matchStr = tree[1]
|
||||||
|
elif isinstance(tree[1], list):
|
||||||
|
matchStr = tree[1][0]
|
||||||
|
if matchStr:
|
||||||
|
if matchStr.startswith('"') and matchStr.endswith('"'):
|
||||||
|
matchStr = matchStr[1:]
|
||||||
|
matchStr = matchStr[:len(matchStr) - 1]
|
||||||
|
return matchStr.lower() in url
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalAnd(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" AND
|
||||||
|
"""
|
||||||
|
if len(tree) < 3:
|
||||||
|
return False
|
||||||
|
for argIndex in range(1, len(tree)):
|
||||||
|
argValue = False
|
||||||
|
if isinstance(tree[argIndex], str):
|
||||||
|
argValue = (tree[argIndex] in hashtags)
|
||||||
|
elif isinstance(tree[argIndex], list):
|
||||||
|
argValue = hashtagRuleResolve(tree[argIndex],
|
||||||
|
hashtags, moderated,
|
||||||
|
content, url)
|
||||||
|
if not argValue:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalOr(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" OR
|
||||||
|
"""
|
||||||
|
if len(tree) < 3:
|
||||||
|
return False
|
||||||
|
for argIndex in range(1, len(tree)):
|
||||||
|
argValue = False
|
||||||
|
if isinstance(tree[argIndex], str):
|
||||||
|
argValue = (tree[argIndex] in hashtags)
|
||||||
|
elif isinstance(tree[argIndex], list):
|
||||||
|
argValue = hashtagRuleResolve(tree[argIndex],
|
||||||
|
hashtags, moderated,
|
||||||
|
content, url)
|
||||||
|
if argValue:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagLogicalXor(tree: [], hashtags: [], moderated: bool,
|
||||||
|
content: str, url: str) -> bool:
|
||||||
|
""" XOR
|
||||||
|
"""
|
||||||
|
if len(tree) < 3:
|
||||||
|
return False
|
||||||
|
trueCtr = 0
|
||||||
|
for argIndex in range(1, len(tree)):
|
||||||
|
argValue = False
|
||||||
|
if isinstance(tree[argIndex], str):
|
||||||
|
argValue = (tree[argIndex] in hashtags)
|
||||||
|
elif isinstance(tree[argIndex], list):
|
||||||
|
argValue = hashtagRuleResolve(tree[argIndex],
|
||||||
|
hashtags, moderated,
|
||||||
|
content, url)
|
||||||
|
if argValue:
|
||||||
|
trueCtr += 1
|
||||||
|
if trueCtr == 1:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
|
def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
|
||||||
content: str, url: str) -> bool:
|
content: str, url: str) -> bool:
|
||||||
"""Returns whether the tree for a hashtag rule evaluates to true or false
|
"""Returns whether the tree for a hashtag rule evaluates to true or false
|
||||||
|
|
@ -82,79 +196,17 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if tree[0] == 'not':
|
if tree[0] == 'not':
|
||||||
if len(tree) == 2:
|
return _hashtagLogicalNot(tree, hashtags, moderated, content, url)
|
||||||
if isinstance(tree[1], str):
|
|
||||||
return tree[1] not in hashtags
|
|
||||||
elif isinstance(tree[1], list):
|
|
||||||
return not hashtagRuleResolve(tree[1], hashtags, moderated,
|
|
||||||
content, url)
|
|
||||||
elif tree[0] == 'contains':
|
elif tree[0] == 'contains':
|
||||||
if len(tree) == 2:
|
return _hashtagLogicalContains(tree, hashtags, moderated, content, url)
|
||||||
matchStr = None
|
|
||||||
if isinstance(tree[1], str):
|
|
||||||
matchStr = tree[1]
|
|
||||||
elif isinstance(tree[1], list):
|
|
||||||
matchStr = tree[1][0]
|
|
||||||
if matchStr:
|
|
||||||
if matchStr.startswith('"') and matchStr.endswith('"'):
|
|
||||||
matchStr = matchStr[1:]
|
|
||||||
matchStr = matchStr[:len(matchStr) - 1]
|
|
||||||
matchStrLower = matchStr.lower()
|
|
||||||
contentWithoutTags = content.replace('#' + matchStrLower, '')
|
|
||||||
return matchStrLower in contentWithoutTags
|
|
||||||
elif tree[0] == 'from':
|
elif tree[0] == 'from':
|
||||||
if len(tree) == 2:
|
return _hashtagLogicalFrom(tree, hashtags, moderated, content, url)
|
||||||
matchStr = None
|
|
||||||
if isinstance(tree[1], str):
|
|
||||||
matchStr = tree[1]
|
|
||||||
elif isinstance(tree[1], list):
|
|
||||||
matchStr = tree[1][0]
|
|
||||||
if matchStr:
|
|
||||||
if matchStr.startswith('"') and matchStr.endswith('"'):
|
|
||||||
matchStr = matchStr[1:]
|
|
||||||
matchStr = matchStr[:len(matchStr) - 1]
|
|
||||||
return matchStr.lower() in url
|
|
||||||
elif tree[0] == 'and':
|
elif tree[0] == 'and':
|
||||||
if len(tree) >= 3:
|
return _hashtagLogicalAnd(tree, hashtags, moderated, content, url)
|
||||||
for argIndex in range(1, len(tree)):
|
|
||||||
argValue = False
|
|
||||||
if isinstance(tree[argIndex], str):
|
|
||||||
argValue = (tree[argIndex] in hashtags)
|
|
||||||
elif isinstance(tree[argIndex], list):
|
|
||||||
argValue = hashtagRuleResolve(tree[argIndex],
|
|
||||||
hashtags, moderated,
|
|
||||||
content, url)
|
|
||||||
if not argValue:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
elif tree[0] == 'or':
|
elif tree[0] == 'or':
|
||||||
if len(tree) >= 3:
|
return _hashtagLogicalOr(tree, hashtags, moderated, content, url)
|
||||||
for argIndex in range(1, len(tree)):
|
|
||||||
argValue = False
|
|
||||||
if isinstance(tree[argIndex], str):
|
|
||||||
argValue = (tree[argIndex] in hashtags)
|
|
||||||
elif isinstance(tree[argIndex], list):
|
|
||||||
argValue = hashtagRuleResolve(tree[argIndex],
|
|
||||||
hashtags, moderated,
|
|
||||||
content, url)
|
|
||||||
if argValue:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
elif tree[0] == 'xor':
|
elif tree[0] == 'xor':
|
||||||
if len(tree) >= 3:
|
return _hashtagLogicalXor(tree, hashtags, moderated, content, url)
|
||||||
trueCtr = 0
|
|
||||||
for argIndex in range(1, len(tree)):
|
|
||||||
argValue = False
|
|
||||||
if isinstance(tree[argIndex], str):
|
|
||||||
argValue = (tree[argIndex] in hashtags)
|
|
||||||
elif isinstance(tree[argIndex], list):
|
|
||||||
argValue = hashtagRuleResolve(tree[argIndex],
|
|
||||||
hashtags, moderated,
|
|
||||||
content, url)
|
|
||||||
if argValue:
|
|
||||||
trueCtr += 1
|
|
||||||
if trueCtr == 1:
|
|
||||||
return True
|
|
||||||
elif tree[0].startswith('#') and len(tree) == 1:
|
elif tree[0].startswith('#') and len(tree) == 1:
|
||||||
return tree[0] in hashtags
|
return tree[0] in hashtags
|
||||||
elif tree[0].startswith('moderated'):
|
elif tree[0].startswith('moderated'):
|
||||||
|
|
@ -225,6 +277,87 @@ def hashtagRuleTree(operators: [],
|
||||||
return tree
|
return tree
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
|
||||||
|
postJsonObject: {},
|
||||||
|
actionStr: str, hashtags: []) -> None:
|
||||||
|
"""Adds a hashtag via a hashtag rule
|
||||||
|
"""
|
||||||
|
addHashtag = actionStr.split('add ', 1)[1].strip()
|
||||||
|
if not addHashtag.startswith('#'):
|
||||||
|
return
|
||||||
|
|
||||||
|
if addHashtag not in hashtags:
|
||||||
|
hashtags.append(addHashtag)
|
||||||
|
htId = addHashtag.replace('#', '')
|
||||||
|
if not validHashTag(htId):
|
||||||
|
return
|
||||||
|
|
||||||
|
hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
|
||||||
|
newTag = {
|
||||||
|
'href': hashtagUrl,
|
||||||
|
'name': addHashtag,
|
||||||
|
'type': 'Hashtag'
|
||||||
|
}
|
||||||
|
# does the tag already exist?
|
||||||
|
addTagObject = None
|
||||||
|
for t in postJsonObject['object']['tag']:
|
||||||
|
if t.get('type') and t.get('name'):
|
||||||
|
if t['type'] == 'Hashtag' and \
|
||||||
|
t['name'] == addHashtag:
|
||||||
|
addTagObject = t
|
||||||
|
break
|
||||||
|
# append the tag if it wasn't found
|
||||||
|
if not addTagObject:
|
||||||
|
postJsonObject['object']['tag'].append(newTag)
|
||||||
|
# add corresponding html to the post content
|
||||||
|
hashtagHtml = \
|
||||||
|
" <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
|
||||||
|
"rel=\"tag\">#<span>" + htId + "</span></a>"
|
||||||
|
content = postJsonObject['object']['content']
|
||||||
|
if hashtagHtml in content:
|
||||||
|
return
|
||||||
|
|
||||||
|
if content.endswith('</p>'):
|
||||||
|
content = \
|
||||||
|
content[:len(content) - len('</p>')] + \
|
||||||
|
hashtagHtml + '</p>'
|
||||||
|
else:
|
||||||
|
content += hashtagHtml
|
||||||
|
postJsonObject['object']['content'] = content
|
||||||
|
storeHashTags(baseDir, 'news', postJsonObject)
|
||||||
|
|
||||||
|
|
||||||
|
def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
|
||||||
|
actionStr: str, hashtags: []) -> None:
|
||||||
|
"""Removes a hashtag via a hashtag rule
|
||||||
|
"""
|
||||||
|
rmHashtag = actionStr.split('remove ', 1)[1].strip()
|
||||||
|
if not rmHashtag.startswith('#'):
|
||||||
|
return
|
||||||
|
|
||||||
|
if rmHashtag in hashtags:
|
||||||
|
hashtags.remove(rmHashtag)
|
||||||
|
htId = rmHashtag.replace('#', '')
|
||||||
|
hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
|
||||||
|
# remove tag html from the post content
|
||||||
|
hashtagHtml = \
|
||||||
|
"<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
|
||||||
|
"rel=\"tag\">#<span>" + htId + "</span></a>"
|
||||||
|
content = postJsonObject['object']['content']
|
||||||
|
if hashtagHtml in content:
|
||||||
|
content = content.replace(hashtagHtml, '').replace(' ', ' ')
|
||||||
|
postJsonObject['object']['content'] = content
|
||||||
|
rmTagObject = None
|
||||||
|
for t in postJsonObject['object']['tag']:
|
||||||
|
if t.get('type') and t.get('name'):
|
||||||
|
if t['type'] == 'Hashtag' and \
|
||||||
|
t['name'] == rmHashtag:
|
||||||
|
rmTagObject = t
|
||||||
|
break
|
||||||
|
if rmTagObject:
|
||||||
|
postJsonObject['object']['tag'].remove(rmTagObject)
|
||||||
|
|
||||||
|
|
||||||
def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
|
def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
|
||||||
hashtags: [], httpPrefix: str,
|
hashtags: [], httpPrefix: str,
|
||||||
domain: str, port: int,
|
domain: str, port: int,
|
||||||
|
|
@ -273,83 +406,16 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
|
||||||
# the condition matches, so do something
|
# the condition matches, so do something
|
||||||
actionStr = ruleStr.split(' then ')[1].strip()
|
actionStr = ruleStr.split(' then ')[1].strip()
|
||||||
|
|
||||||
# add a hashtag
|
|
||||||
if actionStr.startswith('add '):
|
if actionStr.startswith('add '):
|
||||||
addHashtag = actionStr.split('add ', 1)[1].strip()
|
# add a hashtag
|
||||||
if addHashtag.startswith('#'):
|
_hashtagAdd(baseDir, httpPrefix, domainFull,
|
||||||
if addHashtag not in hashtags:
|
postJsonObject, actionStr, hashtags)
|
||||||
hashtags.append(addHashtag)
|
elif actionStr.startswith('remove '):
|
||||||
htId = addHashtag.replace('#', '')
|
# remove a hashtag
|
||||||
if validHashTag(htId):
|
_hashtagRemove(httpPrefix, domainFull, postJsonObject,
|
||||||
hashtagUrl = \
|
actionStr, hashtags)
|
||||||
httpPrefix + "://" + domainFull + "/tags/" + htId
|
elif actionStr.startswith('block') or actionStr.startswith('drop'):
|
||||||
newTag = {
|
# Block this item
|
||||||
'href': hashtagUrl,
|
|
||||||
'name': addHashtag,
|
|
||||||
'type': 'Hashtag'
|
|
||||||
}
|
|
||||||
# does the tag already exist?
|
|
||||||
addTagObject = None
|
|
||||||
for t in postJsonObject['object']['tag']:
|
|
||||||
if t.get('type') and t.get('name'):
|
|
||||||
if t['type'] == 'Hashtag' and \
|
|
||||||
t['name'] == addHashtag:
|
|
||||||
addTagObject = t
|
|
||||||
break
|
|
||||||
# append the tag if it wasn't found
|
|
||||||
if not addTagObject:
|
|
||||||
postJsonObject['object']['tag'].append(newTag)
|
|
||||||
# add corresponding html to the post content
|
|
||||||
hashtagHtml = \
|
|
||||||
" <a href=\"" + hashtagUrl + \
|
|
||||||
"\" class=\"addedHashtag\" " + \
|
|
||||||
"rel=\"tag\">#<span>" + \
|
|
||||||
htId + "</span></a>"
|
|
||||||
content = postJsonObject['object']['content']
|
|
||||||
if hashtagHtml not in content:
|
|
||||||
if content.endswith('</p>'):
|
|
||||||
content = \
|
|
||||||
content[:len(content) - len('</p>')] + \
|
|
||||||
hashtagHtml + '</p>'
|
|
||||||
else:
|
|
||||||
content += hashtagHtml
|
|
||||||
postJsonObject['object']['content'] = content
|
|
||||||
storeHashTags(baseDir, 'news', postJsonObject)
|
|
||||||
# actionOccurred = True
|
|
||||||
|
|
||||||
# remove a hashtag
|
|
||||||
if actionStr.startswith('remove '):
|
|
||||||
rmHashtag = actionStr.split('remove ', 1)[1].strip()
|
|
||||||
if rmHashtag.startswith('#'):
|
|
||||||
if rmHashtag in hashtags:
|
|
||||||
hashtags.remove(rmHashtag)
|
|
||||||
htId = rmHashtag.replace('#', '')
|
|
||||||
hashtagUrl = \
|
|
||||||
httpPrefix + "://" + domainFull + "/tags/" + htId
|
|
||||||
# remove tag html from the post content
|
|
||||||
hashtagHtml = \
|
|
||||||
"<a href=\"" + hashtagUrl + \
|
|
||||||
"\" class=\"addedHashtag\" " + \
|
|
||||||
"rel=\"tag\">#<span>" + \
|
|
||||||
htId + "</span></a>"
|
|
||||||
content = postJsonObject['object']['content']
|
|
||||||
if hashtagHtml in content:
|
|
||||||
content = \
|
|
||||||
content.replace(hashtagHtml, '').replace(' ', ' ')
|
|
||||||
postJsonObject['object']['content'] = content
|
|
||||||
rmTagObject = None
|
|
||||||
for t in postJsonObject['object']['tag']:
|
|
||||||
if t.get('type') and t.get('name'):
|
|
||||||
if t['type'] == 'Hashtag' and \
|
|
||||||
t['name'] == rmHashtag:
|
|
||||||
rmTagObject = t
|
|
||||||
break
|
|
||||||
if rmTagObject:
|
|
||||||
postJsonObject['object']['tag'].remove(rmTagObject)
|
|
||||||
# actionOccurred = True
|
|
||||||
|
|
||||||
# Block this item
|
|
||||||
if actionStr.startswith('block') or actionStr.startswith('drop'):
|
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
|
||||||
35
newswire.py
35
newswire.py
|
|
@ -47,21 +47,25 @@ def rss2Header(httpPrefix: str,
|
||||||
title: str, translate: {}) -> str:
|
title: str, translate: {}) -> str:
|
||||||
"""Header for an RSS 2.0 feed
|
"""Header for an RSS 2.0 feed
|
||||||
"""
|
"""
|
||||||
rssStr = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
|
rssStr = \
|
||||||
rssStr += "<rss version=\"2.0\">"
|
"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + \
|
||||||
rssStr += '<channel>'
|
"<rss version=\"2.0\">" + \
|
||||||
|
'<channel>'
|
||||||
|
|
||||||
if title.startswith('News'):
|
if title.startswith('News'):
|
||||||
rssStr += ' <title>Newswire</title>'
|
rssStr += \
|
||||||
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
|
' <title>Newswire</title>' + \
|
||||||
|
' <link>' + httpPrefix + '://' + domainFull + \
|
||||||
'/newswire.xml' + '</link>'
|
'/newswire.xml' + '</link>'
|
||||||
elif title.startswith('Site'):
|
elif title.startswith('Site'):
|
||||||
rssStr += ' <title>' + domainFull + '</title>'
|
rssStr += \
|
||||||
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
|
' <title>' + domainFull + '</title>' + \
|
||||||
|
' <link>' + httpPrefix + '://' + domainFull + \
|
||||||
'/blog/rss.xml' + '</link>'
|
'/blog/rss.xml' + '</link>'
|
||||||
else:
|
else:
|
||||||
rssStr += ' <title>' + translate[title] + '</title>'
|
rssStr += \
|
||||||
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
|
' <title>' + translate[title] + '</title>' + \
|
||||||
|
' <link>' + httpPrefix + '://' + domainFull + \
|
||||||
'/users/' + nickname + '/rss.xml' + '</link>'
|
'/users/' + nickname + '/rss.xml' + '</link>'
|
||||||
return rssStr
|
return rssStr
|
||||||
|
|
||||||
|
|
@ -69,8 +73,7 @@ def rss2Header(httpPrefix: str,
|
||||||
def rss2Footer() -> str:
|
def rss2Footer() -> str:
|
||||||
"""Footer for an RSS 2.0 feed
|
"""Footer for an RSS 2.0 feed
|
||||||
"""
|
"""
|
||||||
rssStr = '</channel>'
|
rssStr = '</channel></rss>'
|
||||||
rssStr += '</rss>'
|
|
||||||
return rssStr
|
return rssStr
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -815,8 +818,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('WARN: Unable to convert date ' + published + ' ' + str(e))
|
print('WARN: Unable to convert date ' + published + ' ' + str(e))
|
||||||
continue
|
continue
|
||||||
rssStr += '<item>\n'
|
rssStr += \
|
||||||
rssStr += ' <title>' + fields[0] + '</title>\n'
|
'<item>\n' + \
|
||||||
|
' <title>' + fields[0] + '</title>\n'
|
||||||
description = removeHtml(firstParagraphFromString(fields[4]))
|
description = removeHtml(firstParagraphFromString(fields[4]))
|
||||||
rssStr += ' <description>' + description + '</description>\n'
|
rssStr += ' <description>' + description + '</description>\n'
|
||||||
url = fields[1]
|
url = fields[1]
|
||||||
|
|
@ -826,8 +830,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
|
||||||
rssStr += ' <link>' + url + '</link>\n'
|
rssStr += ' <link>' + url + '</link>\n'
|
||||||
|
|
||||||
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
|
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
|
||||||
rssStr += ' <pubDate>' + rssDateStr + '</pubDate>\n'
|
rssStr += \
|
||||||
rssStr += '</item>\n'
|
' <pubDate>' + rssDateStr + '</pubDate>\n' + \
|
||||||
|
'</item>\n'
|
||||||
rssStr += rss2Footer()
|
rssStr += rss2Footer()
|
||||||
return rssStr
|
return rssStr
|
||||||
|
|
||||||
|
|
|
||||||
17
outbox.py
17
outbox.py
|
|
@ -311,8 +311,7 @@ def postMessageToOutbox(session, translate: {},
|
||||||
|
|
||||||
permittedOutboxTypes = ('Create', 'Announce', 'Like', 'Follow', 'Undo',
|
permittedOutboxTypes = ('Create', 'Announce', 'Like', 'Follow', 'Undo',
|
||||||
'Update', 'Add', 'Remove', 'Block', 'Delete',
|
'Update', 'Add', 'Remove', 'Block', 'Delete',
|
||||||
'Skill', 'Add', 'Remove', 'Event',
|
'Skill', 'Ignore')
|
||||||
'Ignore')
|
|
||||||
if messageJson['type'] not in permittedOutboxTypes:
|
if messageJson['type'] not in permittedOutboxTypes:
|
||||||
if debug:
|
if debug:
|
||||||
print('DEBUG: POST to outbox - ' + messageJson['type'] +
|
print('DEBUG: POST to outbox - ' + messageJson['type'] +
|
||||||
|
|
@ -369,14 +368,12 @@ def postMessageToOutbox(session, translate: {},
|
||||||
if os.path.isfile(citationsFilename):
|
if os.path.isfile(citationsFilename):
|
||||||
os.remove(citationsFilename)
|
os.remove(citationsFilename)
|
||||||
|
|
||||||
if messageJson['type'] == 'Create' or \
|
# The following activity types get added to the index files
|
||||||
messageJson['type'] == 'Question' or \
|
indexedActivities = (
|
||||||
messageJson['type'] == 'Note' or \
|
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
|
||||||
messageJson['type'] == 'EncryptedMessage' or \
|
'Patch', 'Announce'
|
||||||
messageJson['type'] == 'Article' or \
|
)
|
||||||
messageJson['type'] == 'Event' or \
|
if messageJson['type'] in indexedActivities:
|
||||||
messageJson['type'] == 'Patch' or \
|
|
||||||
messageJson['type'] == 'Announce':
|
|
||||||
indexes = [outboxName, "inbox"]
|
indexes = [outboxName, "inbox"]
|
||||||
selfActor = \
|
selfActor = \
|
||||||
httpPrefix + '://' + domainFull + '/users/' + postToNickname
|
httpPrefix + '://' + domainFull + '/users/' + postToNickname
|
||||||
|
|
|
||||||
89
person.py
89
person.py
|
|
@ -49,6 +49,7 @@ from utils import refreshNewswire
|
||||||
from utils import getProtocolPrefixes
|
from utils import getProtocolPrefixes
|
||||||
from utils import hasUsersPath
|
from utils import hasUsersPath
|
||||||
from utils import getImageExtensions
|
from utils import getImageExtensions
|
||||||
|
from utils import isImageFile
|
||||||
from session import createSession
|
from session import createSession
|
||||||
from session import getJson
|
from session import getJson
|
||||||
from webfinger import webfingerHandle
|
from webfinger import webfingerHandle
|
||||||
|
|
@ -84,11 +85,7 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
|
||||||
image for the given person
|
image for the given person
|
||||||
"""
|
"""
|
||||||
imageFilename = imageFilename.replace('\n', '').replace('\r', '')
|
imageFilename = imageFilename.replace('\n', '').replace('\r', '')
|
||||||
if not (imageFilename.endswith('.png') or
|
if not isImageFile(imageFilename):
|
||||||
imageFilename.endswith('.jpg') or
|
|
||||||
imageFilename.endswith('.jpeg') or
|
|
||||||
imageFilename.endswith('.svg') or
|
|
||||||
imageFilename.endswith('.gif')):
|
|
||||||
print('Profile image must be png, jpg, gif or svg format')
|
print('Profile image must be png, jpg, gif or svg format')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -119,10 +116,16 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
|
||||||
imageFilename.endswith('.jpeg'):
|
imageFilename.endswith('.jpeg'):
|
||||||
mediaType = 'image/jpeg'
|
mediaType = 'image/jpeg'
|
||||||
iconFilename = iconFilenameBase + '.jpg'
|
iconFilename = iconFilenameBase + '.jpg'
|
||||||
if imageFilename.endswith('.gif'):
|
elif imageFilename.endswith('.gif'):
|
||||||
mediaType = 'image/gif'
|
mediaType = 'image/gif'
|
||||||
iconFilename = iconFilenameBase + '.gif'
|
iconFilename = iconFilenameBase + '.gif'
|
||||||
if imageFilename.endswith('.svg'):
|
elif imageFilename.endswith('.webp'):
|
||||||
|
mediaType = 'image/webp'
|
||||||
|
iconFilename = iconFilenameBase + '.webp'
|
||||||
|
elif imageFilename.endswith('.avif'):
|
||||||
|
mediaType = 'image/avif'
|
||||||
|
iconFilename = iconFilenameBase + '.avif'
|
||||||
|
elif imageFilename.endswith('.svg'):
|
||||||
mediaType = 'image/svg+xml'
|
mediaType = 'image/svg+xml'
|
||||||
iconFilename = iconFilenameBase + '.svg'
|
iconFilename = iconFilenameBase + '.svg'
|
||||||
profileFilename = baseDir + '/accounts/' + handle + '/' + iconFilename
|
profileFilename = baseDir + '/accounts/' + handle + '/' + iconFilename
|
||||||
|
|
@ -593,17 +596,15 @@ def personUpgradeActor(baseDir: str, personJson: {},
|
||||||
# if the older skills format is being used then switch
|
# if the older skills format is being used then switch
|
||||||
# to the new one
|
# to the new one
|
||||||
if not personJson.get('hasOccupation'):
|
if not personJson.get('hasOccupation'):
|
||||||
personJson['hasOccupation'] = [
|
personJson['hasOccupation'] = [{
|
||||||
{
|
'@type': 'Occupation',
|
||||||
'@type': 'Occupation',
|
'name': occupationName,
|
||||||
'name': occupationName,
|
"occupationLocation": {
|
||||||
"occupationLocation": {
|
"@type": "City",
|
||||||
"@type": "City",
|
"name": "Fediverse"
|
||||||
"name": "Fediverse"
|
},
|
||||||
},
|
'skills': []
|
||||||
'skills': []
|
}]
|
||||||
}
|
|
||||||
]
|
|
||||||
updateActor = True
|
updateActor = True
|
||||||
|
|
||||||
# remove the old skills format
|
# remove the old skills format
|
||||||
|
|
@ -618,17 +619,15 @@ def personUpgradeActor(baseDir: str, personJson: {},
|
||||||
updateActor = True
|
updateActor = True
|
||||||
|
|
||||||
if not isinstance(personJson['hasOccupation'], list):
|
if not isinstance(personJson['hasOccupation'], list):
|
||||||
personJson['hasOccupation'] = [
|
personJson['hasOccupation'] = [{
|
||||||
{
|
'@type': 'Occupation',
|
||||||
'@type': 'Occupation',
|
'name': occupationName,
|
||||||
'name': occupationName,
|
'occupationLocation': {
|
||||||
'occupationLocation': {
|
'@type': 'City',
|
||||||
'@type': 'City',
|
'name': 'Fediverse'
|
||||||
'name': 'Fediverse'
|
},
|
||||||
},
|
'skills': []
|
||||||
'skills': []
|
}]
|
||||||
}
|
|
||||||
]
|
|
||||||
updateActor = True
|
updateActor = True
|
||||||
else:
|
else:
|
||||||
# add location if it is missing
|
# add location if it is missing
|
||||||
|
|
@ -1212,27 +1211,17 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool,
|
||||||
for prefix in prefixes:
|
for prefix in prefixes:
|
||||||
handle = handle.replace(prefix, '')
|
handle = handle.replace(prefix, '')
|
||||||
handle = handle.replace('/@', '/users/')
|
handle = handle.replace('/@', '/users/')
|
||||||
if '/users/' in handle:
|
paths = (
|
||||||
nickname = handle.split('/users/')[1]
|
'/users/', '/profile/', '/channel/', '/accounts/', '/u/'
|
||||||
nickname = nickname.replace('\n', '').replace('\r', '')
|
)
|
||||||
domain = handle.split('/users/')[0]
|
userPathFound = False
|
||||||
elif '/profile/' in handle:
|
for userPath in paths:
|
||||||
nickname = handle.split('/profile/')[1]
|
if userPath in handle:
|
||||||
nickname = nickname.replace('\n', '').replace('\r', '')
|
nickname = handle.split(userPath)[1]
|
||||||
domain = handle.split('/profile/')[0]
|
nickname = nickname.replace('\n', '').replace('\r', '')
|
||||||
elif '/channel/' in handle:
|
domain = handle.split(userPath)[0]
|
||||||
nickname = handle.split('/channel/')[1]
|
break
|
||||||
nickname = nickname.replace('\n', '').replace('\r', '')
|
if not userPathFound and '://' in originalHandle:
|
||||||
domain = handle.split('/channel/')[0]
|
|
||||||
elif '/accounts/' in handle:
|
|
||||||
nickname = handle.split('/accounts/')[1]
|
|
||||||
nickname = nickname.replace('\n', '').replace('\r', '')
|
|
||||||
domain = handle.split('/accounts/')[0]
|
|
||||||
elif '/u/' in handle:
|
|
||||||
nickname = handle.split('/u/')[1]
|
|
||||||
nickname = nickname.replace('\n', '').replace('\r', '')
|
|
||||||
domain = handle.split('/u/')[0]
|
|
||||||
elif '://' in originalHandle:
|
|
||||||
domain = originalHandle.split('://')[1]
|
domain = originalHandle.split('://')[1]
|
||||||
if '/' in domain:
|
if '/' in domain:
|
||||||
domain = domain.split('/')[0]
|
domain = domain.split('/')[0]
|
||||||
|
|
|
||||||
100
posts.py
100
posts.py
|
|
@ -492,9 +492,9 @@ def _updateWordFrequency(content: str, wordFrequency: {}) -> None:
|
||||||
that they appear
|
that they appear
|
||||||
"""
|
"""
|
||||||
plainText = removeHtml(content)
|
plainText = removeHtml(content)
|
||||||
plainText = plainText.replace('.', ' ')
|
removeChars = ('.', ';', '?')
|
||||||
plainText = plainText.replace(';', ' ')
|
for ch in removeChars:
|
||||||
plainText = plainText.replace('?', ' ')
|
plainText = plainText.replace(ch, ' ')
|
||||||
wordsList = plainText.split(' ')
|
wordsList = plainText.split(' ')
|
||||||
commonWords = (
|
commonWords = (
|
||||||
'that', 'some', 'about', 'then', 'they', 'were',
|
'that', 'some', 'about', 'then', 'they', 'were',
|
||||||
|
|
@ -1656,29 +1656,30 @@ def getMentionedPeople(baseDir: str, httpPrefix: str,
|
||||||
mentions = []
|
mentions = []
|
||||||
words = content.split(' ')
|
words = content.split(' ')
|
||||||
for wrd in words:
|
for wrd in words:
|
||||||
if wrd.startswith('@'):
|
if not wrd.startswith('@'):
|
||||||
handle = wrd[1:]
|
continue
|
||||||
if debug:
|
handle = wrd[1:]
|
||||||
print('DEBUG: mentioned handle ' + handle)
|
if debug:
|
||||||
if '@' not in handle:
|
print('DEBUG: mentioned handle ' + handle)
|
||||||
handle = handle + '@' + domain
|
if '@' not in handle:
|
||||||
if not os.path.isdir(baseDir + '/accounts/' + handle):
|
handle = handle + '@' + domain
|
||||||
continue
|
if not os.path.isdir(baseDir + '/accounts/' + handle):
|
||||||
else:
|
|
||||||
externalDomain = handle.split('@')[1]
|
|
||||||
if not ('.' in externalDomain or
|
|
||||||
externalDomain == 'localhost'):
|
|
||||||
continue
|
|
||||||
mentionedNickname = handle.split('@')[0]
|
|
||||||
mentionedDomain = handle.split('@')[1].strip('\n').strip('\r')
|
|
||||||
if ':' in mentionedDomain:
|
|
||||||
mentionedDomain = removeDomainPort(mentionedDomain)
|
|
||||||
if not validNickname(mentionedDomain, mentionedNickname):
|
|
||||||
continue
|
continue
|
||||||
actor = \
|
else:
|
||||||
httpPrefix + '://' + handle.split('@')[1] + \
|
externalDomain = handle.split('@')[1]
|
||||||
'/users/' + mentionedNickname
|
if not ('.' in externalDomain or
|
||||||
mentions.append(actor)
|
externalDomain == 'localhost'):
|
||||||
|
continue
|
||||||
|
mentionedNickname = handle.split('@')[0]
|
||||||
|
mentionedDomain = handle.split('@')[1].strip('\n').strip('\r')
|
||||||
|
if ':' in mentionedDomain:
|
||||||
|
mentionedDomain = removeDomainPort(mentionedDomain)
|
||||||
|
if not validNickname(mentionedDomain, mentionedNickname):
|
||||||
|
continue
|
||||||
|
actor = \
|
||||||
|
httpPrefix + '://' + handle.split('@')[1] + \
|
||||||
|
'/users/' + mentionedNickname
|
||||||
|
mentions.append(actor)
|
||||||
return mentions
|
return mentions
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -2140,14 +2141,15 @@ def groupFollowersByDomain(baseDir: str, nickname: str, domain: str) -> {}:
|
||||||
grouped = {}
|
grouped = {}
|
||||||
with open(followersFilename, "r") as f:
|
with open(followersFilename, "r") as f:
|
||||||
for followerHandle in f:
|
for followerHandle in f:
|
||||||
if '@' in followerHandle:
|
if '@' not in followerHandle:
|
||||||
fHandle = \
|
continue
|
||||||
followerHandle.strip().replace('\n', '').replace('\r', '')
|
fHandle = \
|
||||||
followerDomain = fHandle.split('@')[1]
|
followerHandle.strip().replace('\n', '').replace('\r', '')
|
||||||
if not grouped.get(followerDomain):
|
followerDomain = fHandle.split('@')[1]
|
||||||
grouped[followerDomain] = [fHandle]
|
if not grouped.get(followerDomain):
|
||||||
else:
|
grouped[followerDomain] = [fHandle]
|
||||||
grouped[followerDomain].append(fHandle)
|
else:
|
||||||
|
grouped[followerDomain].append(fHandle)
|
||||||
return grouped
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -2174,9 +2176,9 @@ def _addFollowersToPublicPost(postJsonObject: {}) -> None:
|
||||||
return
|
return
|
||||||
if len(postJsonObject['object']['to']) > 1:
|
if len(postJsonObject['object']['to']) > 1:
|
||||||
return
|
return
|
||||||
if len(postJsonObject['object']['to']) == 0:
|
elif len(postJsonObject['object']['to']) == 0:
|
||||||
return
|
return
|
||||||
if not postJsonObject['object']['to'][0].endswith('#Public'):
|
elif not postJsonObject['object']['to'][0].endswith('#Public'):
|
||||||
return
|
return
|
||||||
if postJsonObject['object'].get('cc'):
|
if postJsonObject['object'].get('cc'):
|
||||||
return
|
return
|
||||||
|
|
@ -2403,6 +2405,20 @@ def addToField(activityType: str, postJsonObject: {},
|
||||||
return postJsonObject, False
|
return postJsonObject, False
|
||||||
|
|
||||||
|
|
||||||
|
def _isProfileUpdate(postJsonObject: {}) -> bool:
|
||||||
|
"""Is the given post a profile update?
|
||||||
|
for actor updates there is no 'to' within the object
|
||||||
|
"""
|
||||||
|
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
|
||||||
|
if (postJsonObject['type'] == 'Update' and
|
||||||
|
(postJsonObject['object']['type'] == 'Person' or
|
||||||
|
postJsonObject['object']['type'] == 'Application' or
|
||||||
|
postJsonObject['object']['type'] == 'Group' or
|
||||||
|
postJsonObject['object']['type'] == 'Service')):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def sendToNamedAddresses(session, baseDir: str,
|
def sendToNamedAddresses(session, baseDir: str,
|
||||||
nickname: str, domain: str,
|
nickname: str, domain: str,
|
||||||
onionDomain: str, i2pDomain: str, port: int,
|
onionDomain: str, i2pDomain: str, port: int,
|
||||||
|
|
@ -2420,16 +2436,10 @@ def sendToNamedAddresses(session, baseDir: str,
|
||||||
return
|
return
|
||||||
isProfileUpdate = False
|
isProfileUpdate = False
|
||||||
if isinstance(postJsonObject['object'], dict):
|
if isinstance(postJsonObject['object'], dict):
|
||||||
# for actor updates there is no 'to' within the object
|
if _isProfileUpdate(postJsonObject):
|
||||||
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
|
# use the original object, which has a 'to'
|
||||||
if (postJsonObject['type'] == 'Update' and
|
recipientsObject = postJsonObject
|
||||||
(postJsonObject['object']['type'] == 'Person' or
|
isProfileUpdate = True
|
||||||
postJsonObject['object']['type'] == 'Application' or
|
|
||||||
postJsonObject['object']['type'] == 'Group' or
|
|
||||||
postJsonObject['object']['type'] == 'Service')):
|
|
||||||
# use the original object, which has a 'to'
|
|
||||||
recipientsObject = postJsonObject
|
|
||||||
isProfileUpdate = True
|
|
||||||
|
|
||||||
if not isProfileUpdate:
|
if not isProfileUpdate:
|
||||||
if not postJsonObject['object'].get('to'):
|
if not postJsonObject['object'].get('to'):
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue