indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon

main
Bob Mottram 2021-07-04 13:51:35 +01:00
parent 882e0eff3c a06186b26e
commit 65884aa33a
5 changed files with 331 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

250
newsdaemon.py
View File

@ -74,22 +74,26 @@ def _removeControlCharacters(content: str) -> str:
return content
def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
def _hashtagLogicalNot(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
"""Returns whether the tree for a hashtag rule evaluates to true or false
""" NOT
"""
if not tree:
if len(tree) != 2:
return False
if tree[0] == 'not':
if len(tree) == 2:
if isinstance(tree[1], str):
return tree[1] not in hashtags
elif isinstance(tree[1], list):
return not hashtagRuleResolve(tree[1], hashtags, moderated,
content, url)
elif tree[0] == 'contains':
if len(tree) == 2:
return not hashtagRuleResolve(tree[1], hashtags,
moderated, content, url)
return False
def _hashtagLogicalContains(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
""" Contains
"""
if len(tree) != 2:
return False
matchStr = None
if isinstance(tree[1], str):
matchStr = tree[1]
@ -102,8 +106,15 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
matchStrLower = matchStr.lower()
contentWithoutTags = content.replace('#' + matchStrLower, '')
return matchStrLower in contentWithoutTags
elif tree[0] == 'from':
if len(tree) == 2:
return False
def _hashtagLogicalFrom(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
""" FROM
"""
if len(tree) != 2:
return False
matchStr = None
if isinstance(tree[1], str):
matchStr = tree[1]
@ -114,8 +125,15 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
matchStr = matchStr[1:]
matchStr = matchStr[:len(matchStr) - 1]
return matchStr.lower() in url
elif tree[0] == 'and':
if len(tree) >= 3:
return False
def _hashtagLogicalAnd(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
""" AND
"""
if len(tree) < 3:
return False
for argIndex in range(1, len(tree)):
argValue = False
if isinstance(tree[argIndex], str):
@ -127,8 +145,14 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
if not argValue:
return False
return True
elif tree[0] == 'or':
if len(tree) >= 3:
def _hashtagLogicalOr(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
""" OR
"""
if len(tree) < 3:
return False
for argIndex in range(1, len(tree)):
argValue = False
if isinstance(tree[argIndex], str):
@ -140,8 +164,14 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
if argValue:
return True
return False
elif tree[0] == 'xor':
if len(tree) >= 3:
def _hashtagLogicalXor(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
""" XOR
"""
if len(tree) < 3:
return False
trueCtr = 0
for argIndex in range(1, len(tree)):
argValue = False
@ -155,6 +185,28 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
trueCtr += 1
if trueCtr == 1:
return True
return False
def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
content: str, url: str) -> bool:
"""Returns whether the tree for a hashtag rule evaluates to true or false
"""
if not tree:
return False
if tree[0] == 'not':
return _hashtagLogicalNot(tree, hashtags, moderated, content, url)
elif tree[0] == 'contains':
return _hashtagLogicalContains(tree, hashtags, moderated, content, url)
elif tree[0] == 'from':
return _hashtagLogicalFrom(tree, hashtags, moderated, content, url)
elif tree[0] == 'and':
return _hashtagLogicalAnd(tree, hashtags, moderated, content, url)
elif tree[0] == 'or':
return _hashtagLogicalOr(tree, hashtags, moderated, content, url)
elif tree[0] == 'xor':
return _hashtagLogicalXor(tree, hashtags, moderated, content, url)
elif tree[0].startswith('#') and len(tree) == 1:
return tree[0] in hashtags
elif tree[0].startswith('moderated'):
@ -225,6 +277,87 @@ def hashtagRuleTree(operators: [],
return tree
def _hashtagAdd(baseDir: str, httpPrefix: str, domainFull: str,
postJsonObject: {},
actionStr: str, hashtags: []) -> None:
"""Adds a hashtag via a hashtag rule
"""
addHashtag = actionStr.split('add ', 1)[1].strip()
if not addHashtag.startswith('#'):
return
if addHashtag not in hashtags:
hashtags.append(addHashtag)
htId = addHashtag.replace('#', '')
if not validHashTag(htId):
return
hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
newTag = {
'href': hashtagUrl,
'name': addHashtag,
'type': 'Hashtag'
}
# does the tag already exist?
addTagObject = None
for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'):
if t['type'] == 'Hashtag' and \
t['name'] == addHashtag:
addTagObject = t
break
# append the tag if it wasn't found
if not addTagObject:
postJsonObject['object']['tag'].append(newTag)
# add corresponding html to the post content
hashtagHtml = \
" <a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content']
if hashtagHtml in content:
return
if content.endswith('</p>'):
content = \
content[:len(content) - len('</p>')] + \
hashtagHtml + '</p>'
else:
content += hashtagHtml
postJsonObject['object']['content'] = content
storeHashTags(baseDir, 'news', postJsonObject)
def _hashtagRemove(httpPrefix: str, domainFull: str, postJsonObject: {},
actionStr: str, hashtags: []) -> None:
"""Removes a hashtag via a hashtag rule
"""
rmHashtag = actionStr.split('remove ', 1)[1].strip()
if not rmHashtag.startswith('#'):
return
if rmHashtag in hashtags:
hashtags.remove(rmHashtag)
htId = rmHashtag.replace('#', '')
hashtagUrl = httpPrefix + "://" + domainFull + "/tags/" + htId
# remove tag html from the post content
hashtagHtml = \
"<a href=\"" + hashtagUrl + "\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + htId + "</span></a>"
content = postJsonObject['object']['content']
if hashtagHtml in content:
content = content.replace(hashtagHtml, '').replace(' ', ' ')
postJsonObject['object']['content'] = content
rmTagObject = None
for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'):
if t['type'] == 'Hashtag' and \
t['name'] == rmHashtag:
rmTagObject = t
break
if rmTagObject:
postJsonObject['object']['tag'].remove(rmTagObject)
def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
hashtags: [], httpPrefix: str,
domain: str, port: int,
@ -273,83 +406,16 @@ def _newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
# the condition matches, so do something
actionStr = ruleStr.split(' then ')[1].strip()
# add a hashtag
if actionStr.startswith('add '):
addHashtag = actionStr.split('add ', 1)[1].strip()
if addHashtag.startswith('#'):
if addHashtag not in hashtags:
hashtags.append(addHashtag)
htId = addHashtag.replace('#', '')
if validHashTag(htId):
hashtagUrl = \
httpPrefix + "://" + domainFull + "/tags/" + htId
newTag = {
'href': hashtagUrl,
'name': addHashtag,
'type': 'Hashtag'
}
# does the tag already exist?
addTagObject = None
for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'):
if t['type'] == 'Hashtag' and \
t['name'] == addHashtag:
addTagObject = t
break
# append the tag if it wasn't found
if not addTagObject:
postJsonObject['object']['tag'].append(newTag)
# add corresponding html to the post content
hashtagHtml = \
" <a href=\"" + hashtagUrl + \
"\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + \
htId + "</span></a>"
content = postJsonObject['object']['content']
if hashtagHtml not in content:
if content.endswith('</p>'):
content = \
content[:len(content) - len('</p>')] + \
hashtagHtml + '</p>'
else:
content += hashtagHtml
postJsonObject['object']['content'] = content
storeHashTags(baseDir, 'news', postJsonObject)
# actionOccurred = True
# add a hashtag
_hashtagAdd(baseDir, httpPrefix, domainFull,
postJsonObject, actionStr, hashtags)
elif actionStr.startswith('remove '):
# remove a hashtag
if actionStr.startswith('remove '):
rmHashtag = actionStr.split('remove ', 1)[1].strip()
if rmHashtag.startswith('#'):
if rmHashtag in hashtags:
hashtags.remove(rmHashtag)
htId = rmHashtag.replace('#', '')
hashtagUrl = \
httpPrefix + "://" + domainFull + "/tags/" + htId
# remove tag html from the post content
hashtagHtml = \
"<a href=\"" + hashtagUrl + \
"\" class=\"addedHashtag\" " + \
"rel=\"tag\">#<span>" + \
htId + "</span></a>"
content = postJsonObject['object']['content']
if hashtagHtml in content:
content = \
content.replace(hashtagHtml, '').replace(' ', ' ')
postJsonObject['object']['content'] = content
rmTagObject = None
for t in postJsonObject['object']['tag']:
if t.get('type') and t.get('name'):
if t['type'] == 'Hashtag' and \
t['name'] == rmHashtag:
rmTagObject = t
break
if rmTagObject:
postJsonObject['object']['tag'].remove(rmTagObject)
# actionOccurred = True
_hashtagRemove(httpPrefix, domainFull, postJsonObject,
actionStr, hashtags)
elif actionStr.startswith('block') or actionStr.startswith('drop'):
# Block this item
if actionStr.startswith('block') or actionStr.startswith('drop'):
return False
return True

35
newswire.py
View File

@ -47,21 +47,25 @@ def rss2Header(httpPrefix: str,
title: str, translate: {}) -> str:
"""Header for an RSS 2.0 feed
"""
rssStr = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
rssStr += "<rss version=\"2.0\">"
rssStr += '<channel>'
rssStr = \
"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" + \
"<rss version=\"2.0\">" + \
'<channel>'
if title.startswith('News'):
rssStr += ' <title>Newswire</title>'
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
rssStr += \
' <title>Newswire</title>' + \
' <link>' + httpPrefix + '://' + domainFull + \
'/newswire.xml' + '</link>'
elif title.startswith('Site'):
rssStr += ' <title>' + domainFull + '</title>'
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
rssStr += \
' <title>' + domainFull + '</title>' + \
' <link>' + httpPrefix + '://' + domainFull + \
'/blog/rss.xml' + '</link>'
else:
rssStr += ' <title>' + translate[title] + '</title>'
rssStr += ' <link>' + httpPrefix + '://' + domainFull + \
rssStr += \
' <title>' + translate[title] + '</title>' + \
' <link>' + httpPrefix + '://' + domainFull + \
'/users/' + nickname + '/rss.xml' + '</link>'
return rssStr
@ -69,8 +73,7 @@ def rss2Header(httpPrefix: str,
def rss2Footer() -> str:
"""Footer for an RSS 2.0 feed
"""
rssStr = '</channel>'
rssStr += '</rss>'
rssStr = '</channel></rss>'
return rssStr
@ -815,8 +818,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
except Exception as e:
print('WARN: Unable to convert date ' + published + ' ' + str(e))
continue
rssStr += '<item>\n'
rssStr += ' <title>' + fields[0] + '</title>\n'
rssStr += \
'<item>\n' + \
' <title>' + fields[0] + '</title>\n'
description = removeHtml(firstParagraphFromString(fields[4]))
rssStr += ' <description>' + description + '</description>\n'
url = fields[1]
@ -826,8 +830,9 @@ def getRSSfromDict(baseDir: str, newswire: {},
rssStr += ' <link>' + url + '</link>\n'
rssDateStr = pubDate.strftime("%a, %d %b %Y %H:%M:%S UT")
rssStr += ' <pubDate>' + rssDateStr + '</pubDate>\n'
rssStr += '</item>\n'
rssStr += \
' <pubDate>' + rssDateStr + '</pubDate>\n' + \
'</item>\n'
rssStr += rss2Footer()
return rssStr

17
outbox.py
View File

@ -311,8 +311,7 @@ def postMessageToOutbox(session, translate: {},
permittedOutboxTypes = ('Create', 'Announce', 'Like', 'Follow', 'Undo',
'Update', 'Add', 'Remove', 'Block', 'Delete',
'Skill', 'Add', 'Remove', 'Event',
'Ignore')
'Skill', 'Ignore')
if messageJson['type'] not in permittedOutboxTypes:
if debug:
print('DEBUG: POST to outbox - ' + messageJson['type'] +
@ -369,14 +368,12 @@ def postMessageToOutbox(session, translate: {},
if os.path.isfile(citationsFilename):
os.remove(citationsFilename)
if messageJson['type'] == 'Create' or \
messageJson['type'] == 'Question' or \
messageJson['type'] == 'Note' or \
messageJson['type'] == 'EncryptedMessage' or \
messageJson['type'] == 'Article' or \
messageJson['type'] == 'Event' or \
messageJson['type'] == 'Patch' or \
messageJson['type'] == 'Announce':
# The following activity types get added to the index files
indexedActivities = (
'Create', 'Question', 'Note', 'EncryptedMessage', 'Article',
'Patch', 'Announce'
)
if messageJson['type'] in indexedActivities:
indexes = [outboxName, "inbox"]
selfActor = \
httpPrefix + '://' + domainFull + '/users/' + postToNickname

59
person.py
View File

@ -49,6 +49,7 @@ from utils import refreshNewswire
from utils import getProtocolPrefixes
from utils import hasUsersPath
from utils import getImageExtensions
from utils import isImageFile
from session import createSession
from session import getJson
from webfinger import webfingerHandle
@ -84,11 +85,7 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
image for the given person
"""
imageFilename = imageFilename.replace('\n', '').replace('\r', '')
if not (imageFilename.endswith('.png') or
imageFilename.endswith('.jpg') or
imageFilename.endswith('.jpeg') or
imageFilename.endswith('.svg') or
imageFilename.endswith('.gif')):
if not isImageFile(imageFilename):
print('Profile image must be png, jpg, gif or svg format')
return False
@ -119,10 +116,16 @@ def setProfileImage(baseDir: str, httpPrefix: str, nickname: str, domain: str,
imageFilename.endswith('.jpeg'):
mediaType = 'image/jpeg'
iconFilename = iconFilenameBase + '.jpg'
if imageFilename.endswith('.gif'):
elif imageFilename.endswith('.gif'):
mediaType = 'image/gif'
iconFilename = iconFilenameBase + '.gif'
if imageFilename.endswith('.svg'):
elif imageFilename.endswith('.webp'):
mediaType = 'image/webp'
iconFilename = iconFilenameBase + '.webp'
elif imageFilename.endswith('.avif'):
mediaType = 'image/avif'
iconFilename = iconFilenameBase + '.avif'
elif imageFilename.endswith('.svg'):
mediaType = 'image/svg+xml'
iconFilename = iconFilenameBase + '.svg'
profileFilename = baseDir + '/accounts/' + handle + '/' + iconFilename
@ -593,8 +596,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
# if the older skills format is being used then switch
# to the new one
if not personJson.get('hasOccupation'):
personJson['hasOccupation'] = [
{
personJson['hasOccupation'] = [{
'@type': 'Occupation',
'name': occupationName,
"occupationLocation": {
@ -602,8 +604,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
"name": "Fediverse"
},
'skills': []
}
]
}]
updateActor = True
# remove the old skills format
@ -618,8 +619,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
updateActor = True
if not isinstance(personJson['hasOccupation'], list):
personJson['hasOccupation'] = [
{
personJson['hasOccupation'] = [{
'@type': 'Occupation',
'name': occupationName,
'occupationLocation': {
@ -627,8 +627,7 @@ def personUpgradeActor(baseDir: str, personJson: {},
'name': 'Fediverse'
},
'skills': []
}
]
}]
updateActor = True
else:
# add location if it is missing
@ -1212,27 +1211,17 @@ def getActorJson(hostDomain: str, handle: str, http: bool, gnunet: bool,
for prefix in prefixes:
handle = handle.replace(prefix, '')
handle = handle.replace('/@', '/users/')
if '/users/' in handle:
nickname = handle.split('/users/')[1]
paths = (
'/users/', '/profile/', '/channel/', '/accounts/', '/u/'
)
userPathFound = False
for userPath in paths:
if userPath in handle:
nickname = handle.split(userPath)[1]
nickname = nickname.replace('\n', '').replace('\r', '')
domain = handle.split('/users/')[0]
elif '/profile/' in handle:
nickname = handle.split('/profile/')[1]
nickname = nickname.replace('\n', '').replace('\r', '')
domain = handle.split('/profile/')[0]
elif '/channel/' in handle:
nickname = handle.split('/channel/')[1]
nickname = nickname.replace('\n', '').replace('\r', '')
domain = handle.split('/channel/')[0]
elif '/accounts/' in handle:
nickname = handle.split('/accounts/')[1]
nickname = nickname.replace('\n', '').replace('\r', '')
domain = handle.split('/accounts/')[0]
elif '/u/' in handle:
nickname = handle.split('/u/')[1]
nickname = nickname.replace('\n', '').replace('\r', '')
domain = handle.split('/u/')[0]
elif '://' in originalHandle:
domain = handle.split(userPath)[0]
break
if not userPathFound and '://' in originalHandle:
domain = originalHandle.split('://')[1]
if '/' in domain:
domain = domain.split('/')[0]

38
posts.py
View File

@ -492,9 +492,9 @@ def _updateWordFrequency(content: str, wordFrequency: {}) -> None:
that they appear
"""
plainText = removeHtml(content)
plainText = plainText.replace('.', ' ')
plainText = plainText.replace(';', ' ')
plainText = plainText.replace('?', ' ')
removeChars = ('.', ';', '?')
for ch in removeChars:
plainText = plainText.replace(ch, ' ')
wordsList = plainText.split(' ')
commonWords = (
'that', 'some', 'about', 'then', 'they', 'were',
@ -1656,7 +1656,8 @@ def getMentionedPeople(baseDir: str, httpPrefix: str,
mentions = []
words = content.split(' ')
for wrd in words:
if wrd.startswith('@'):
if not wrd.startswith('@'):
continue
handle = wrd[1:]
if debug:
print('DEBUG: mentioned handle ' + handle)
@ -2140,7 +2141,8 @@ def groupFollowersByDomain(baseDir: str, nickname: str, domain: str) -> {}:
grouped = {}
with open(followersFilename, "r") as f:
for followerHandle in f:
if '@' in followerHandle:
if '@' not in followerHandle:
continue
fHandle = \
followerHandle.strip().replace('\n', '').replace('\r', '')
followerDomain = fHandle.split('@')[1]
@ -2174,9 +2176,9 @@ def _addFollowersToPublicPost(postJsonObject: {}) -> None:
return
if len(postJsonObject['object']['to']) > 1:
return
if len(postJsonObject['object']['to']) == 0:
elif len(postJsonObject['object']['to']) == 0:
return
if not postJsonObject['object']['to'][0].endswith('#Public'):
elif not postJsonObject['object']['to'][0].endswith('#Public'):
return
if postJsonObject['object'].get('cc'):
return
@ -2403,6 +2405,20 @@ def addToField(activityType: str, postJsonObject: {},
return postJsonObject, False
def _isProfileUpdate(postJsonObject: {}) -> bool:
"""Is the given post a profile update?
for actor updates there is no 'to' within the object
"""
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
if (postJsonObject['type'] == 'Update' and
(postJsonObject['object']['type'] == 'Person' or
postJsonObject['object']['type'] == 'Application' or
postJsonObject['object']['type'] == 'Group' or
postJsonObject['object']['type'] == 'Service')):
return True
return False
def sendToNamedAddresses(session, baseDir: str,
nickname: str, domain: str,
onionDomain: str, i2pDomain: str, port: int,
@ -2420,13 +2436,7 @@ def sendToNamedAddresses(session, baseDir: str,
return
isProfileUpdate = False
if isinstance(postJsonObject['object'], dict):
# for actor updates there is no 'to' within the object
if postJsonObject['object'].get('type') and postJsonObject.get('type'):
if (postJsonObject['type'] == 'Update' and
(postJsonObject['object']['type'] == 'Person' or
postJsonObject['object']['type'] == 'Application' or
postJsonObject['object']['type'] == 'Group' or
postJsonObject['object']['type'] == 'Service')):
if _isProfileUpdate(postJsonObject):
# use the original object, which has a 'to'
recipientsObject = postJsonObject
isProfileUpdate = True