mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Hashtag logic can test for post content

main
Bob Mottram 2020-10-17 19:49:43 +01:00
parent f934f54528
commit dd580be52f
2 changed files with 80 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
newsdaemon.py
View File

@ -73,7 +73,8 @@ def removeControlCharacters(content: str) -> str:
return content return content
def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool) -> bool: def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool,
content: str) -> bool:
"""Returns whether the tree for a hashtag rule evaluates to true or false """Returns whether the tree for a hashtag rule evaluates to true or false
""" """
if not tree: if not tree:
@ -84,7 +85,22 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool) -> bool:
if isinstance(tree[1], str): if isinstance(tree[1], str):
return tree[1] not in hashtags return tree[1] not in hashtags
elif isinstance(tree[1], list): elif isinstance(tree[1], list):
return not hashtagRuleResolve(tree[1], hashtags, moderated) return not hashtagRuleResolve(tree[1], hashtags, moderated,
content)
elif tree[0] == 'contains':
if len(tree) == 2:
if isinstance(tree[1], str):
matchStr = tree[1]
if matchStr.startswith('"') and matchStr.endswith('"'):
matchStr = matchStr[1:]
matchStr = matchStr[:len(matchStr) - 1]
return matchStr in content
elif isinstance(tree[1], list):
matchStr = tree[1][0]
if matchStr.startswith('"') and matchStr.endswith('"'):
matchStr = matchStr[1:]
matchStr = matchStr[:len(matchStr) - 1]
return matchStr in content
elif tree[0] == 'and': elif tree[0] == 'and':
if len(tree) == 3: if len(tree) == 3:
@ -92,13 +108,15 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool) -> bool:
if isinstance(tree[1], str): if isinstance(tree[1], str):
firstArg = (tree[1] in hashtags) firstArg = (tree[1] in hashtags)
elif isinstance(tree[1], list): elif isinstance(tree[1], list):
firstArg = (hashtagRuleResolve(tree[1], hashtags, moderated)) firstArg = (hashtagRuleResolve(tree[1], hashtags, moderated,
content))
secondArg = False secondArg = False
if isinstance(tree[2], str): if isinstance(tree[2], str):
secondArg = (tree[2] in hashtags) secondArg = (tree[2] in hashtags)
elif isinstance(tree[2], list): elif isinstance(tree[2], list):
secondArg = (hashtagRuleResolve(tree[2], hashtags, moderated)) secondArg = (hashtagRuleResolve(tree[2], hashtags, moderated,
content))
return firstArg and secondArg return firstArg and secondArg
elif tree[0] == 'or': elif tree[0] == 'or':
if len(tree) == 3: if len(tree) == 3:
@ -107,18 +125,22 @@ def hashtagRuleResolve(tree: [], hashtags: [], moderated: bool) -> bool:
if isinstance(tree[1], str): if isinstance(tree[1], str):
firstArg = (tree[1] in hashtags) firstArg = (tree[1] in hashtags)
elif isinstance(tree[1], list): elif isinstance(tree[1], list):
firstArg = (hashtagRuleResolve(tree[1], hashtags, moderated)) firstArg = (hashtagRuleResolve(tree[1], hashtags, moderated,
content))
secondArg = False secondArg = False
if isinstance(tree[2], str): if isinstance(tree[2], str):
secondArg = (tree[2] in hashtags) secondArg = (tree[2] in hashtags)
elif isinstance(tree[2], list): elif isinstance(tree[2], list):
secondArg = (hashtagRuleResolve(tree[2], hashtags, moderated)) secondArg = (hashtagRuleResolve(tree[2], hashtags, moderated,
content))
return firstArg or secondArg return firstArg or secondArg
elif tree[0].startswith('#') and len(tree) == 1: elif tree[0].startswith('#') and len(tree) == 1:
return tree[0] in hashtags return tree[0] in hashtags
elif tree[0].startswith('moderated'): elif tree[0].startswith('moderated'):
return moderated return moderated
elif tree[0].startswith('"') and tree[0].endswith('"'):
return True
return False return False
@ -131,12 +153,15 @@ def hashtagRuleTree(operators: [],
""" """
if not operators and conditionsStr: if not operators and conditionsStr:
conditionsStr = conditionsStr.strip() conditionsStr = conditionsStr.strip()
if conditionsStr.startswith('#') or \ isStr = conditionsStr.startswith('"') and conditionsStr.endswith('"')
if conditionsStr.startswith('#') or isStr or \
conditionsStr in operators or \ conditionsStr in operators or \
conditionsStr == 'moderated': conditionsStr == 'moderated' or \
conditionsStr == 'contains':
if conditionsStr.startswith('#'): if conditionsStr.startswith('#'):
if conditionsStr not in tagsInConditions: if conditionsStr not in tagsInConditions:
if ' ' not in conditionsStr: if ' ' not in conditionsStr or \
conditionsStr.startswith('"'):
tagsInConditions.append(conditionsStr) tagsInConditions.append(conditionsStr)
return [conditionsStr.strip()] return [conditionsStr.strip()]
else: else:
@ -145,12 +170,15 @@ def hashtagRuleTree(operators: [],
return None return None
tree = None tree = None
conditionsStr = conditionsStr.strip() conditionsStr = conditionsStr.strip()
if conditionsStr.startswith('#') or \ isStr = conditionsStr.startswith('"') and conditionsStr.endswith('"')
if conditionsStr.startswith('#') or isStr or \
conditionsStr in operators or \ conditionsStr in operators or \
conditionsStr == 'moderated': conditionsStr == 'moderated' or \
conditionsStr == 'contains':
if conditionsStr.startswith('#'): if conditionsStr.startswith('#'):
if conditionsStr not in tagsInConditions: if conditionsStr not in tagsInConditions:
if ' ' not in conditionsStr: if ' ' not in conditionsStr or \
conditionsStr.startswith('"'):
tagsInConditions.append(conditionsStr) tagsInConditions.append(conditionsStr)
tree = [conditionsStr.strip()] tree = [conditionsStr.strip()]
ctr = 0 ctr = 0
@ -195,8 +223,15 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
if port != 80 and port != 443: if port != 80 and port != 443:
domainFull = domain + ':' + str(port) domainFull = domain + ':' + str(port)
# get the full text content of the post
content = ''
if postJsonObject['object'].get('content'):
content += postJsonObject['object']['content']
if postJsonObject['object'].get('summary'):
content += ' ' + postJsonObject['object']['summary']
# actionOccurred = False # actionOccurred = False
operators = ('not', 'and', 'or') operators = ('not', 'and', 'or', 'contains')
for ruleStr in rules: for ruleStr in rules:
if not ruleStr: if not ruleStr:
continue continue
@ -212,7 +247,7 @@ def newswireHashtagProcessing(session, baseDir: str, postJsonObject: {},
# does the rule contain any hashtags? # does the rule contain any hashtags?
if not tagsInConditions: if not tagsInConditions:
continue continue
if not hashtagRuleResolve(tree, hashtags, moderated): if not hashtagRuleResolve(tree, hashtags, moderated, content):
continue continue
# the condition matches, so do something # the condition matches, so do something
actionStr = ruleStr.split(' then ')[1].strip() actionStr = ruleStr.split(' then ')[1].strip()

46
tests.py
View File

@ -2177,8 +2177,9 @@ def testRemoveHtmlTag():
def testHashtagRuleTree(): def testHashtagRuleTree():
print('testHashtagRuleTree') print('testHashtagRuleTree')
operators = ('not', 'and', 'or') operators = ('not', 'and', 'or', 'contains')
content = 'This is a test'
moderated = True moderated = True
conditionsStr = '#foo or #bar' conditionsStr = '#foo or #bar'
tagsInConditions = [] tagsInConditions = []
@ -2187,9 +2188,24 @@ def testHashtagRuleTree():
assert str(tree) == str(['or', ['#foo'], ['#bar']]) assert str(tree) == str(['or', ['#foo'], ['#bar']])
assert str(tagsInConditions) == str(['#foo', '#bar']) assert str(tagsInConditions) == str(['#foo', '#bar'])
hashtags = ['#foo'] hashtags = ['#foo']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#carrot', '#stick'] hashtags = ['#carrot', '#stick']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
content = 'This is a test'
moderated = True
conditionsStr = 'contains "is a" and #foo or #bar'
tagsInConditions = []
tree = hashtagRuleTree(operators, conditionsStr,
tagsInConditions, moderated)
assert str(tree) == \
str(['and', ['contains', ['"is a"']],
['or', ['#foo'], ['#bar']]])
assert str(tagsInConditions) == str(['#foo', '#bar'])
hashtags = ['#foo']
assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#carrot', '#stick']
assert not hashtagRuleResolve(tree, hashtags, moderated, content)
moderated = False moderated = False
conditionsStr = 'not moderated and #foo or #bar' conditionsStr = 'not moderated and #foo or #bar'
@ -2200,9 +2216,9 @@ def testHashtagRuleTree():
str(['not', ['and', ['moderated'], ['or', ['#foo'], ['#bar']]]]) str(['not', ['and', ['moderated'], ['or', ['#foo'], ['#bar']]]])
assert str(tagsInConditions) == str(['#foo', '#bar']) assert str(tagsInConditions) == str(['#foo', '#bar'])
hashtags = ['#foo'] hashtags = ['#foo']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#carrot', '#stick'] hashtags = ['#carrot', '#stick']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
moderated = True moderated = True
conditionsStr = 'moderated and #foo or #bar' conditionsStr = 'moderated and #foo or #bar'
@ -2213,9 +2229,9 @@ def testHashtagRuleTree():
str(['and', ['moderated'], ['or', ['#foo'], ['#bar']]]) str(['and', ['moderated'], ['or', ['#foo'], ['#bar']]])
assert str(tagsInConditions) == str(['#foo', '#bar']) assert str(tagsInConditions) == str(['#foo', '#bar'])
hashtags = ['#foo'] hashtags = ['#foo']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#carrot', '#stick'] hashtags = ['#carrot', '#stick']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
conditionsStr = 'x' conditionsStr = 'x'
tagsInConditions = [] tagsInConditions = []
@ -2224,7 +2240,7 @@ def testHashtagRuleTree():
assert tree is None assert tree is None
assert tagsInConditions == [] assert tagsInConditions == []
hashtags = ['#foo'] hashtags = ['#foo']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
conditionsStr = '#x' conditionsStr = '#x'
tagsInConditions = [] tagsInConditions = []
@ -2233,9 +2249,9 @@ def testHashtagRuleTree():
assert str(tree) == str(['#x']) assert str(tree) == str(['#x'])
assert str(tagsInConditions) == str(['#x']) assert str(tagsInConditions) == str(['#x'])
hashtags = ['#x'] hashtags = ['#x']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#y', '#z'] hashtags = ['#y', '#z']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
conditionsStr = 'not #b' conditionsStr = 'not #b'
tagsInConditions = [] tagsInConditions = []
@ -2244,9 +2260,9 @@ def testHashtagRuleTree():
assert str(tree) == str(['not', ['#b']]) assert str(tree) == str(['not', ['#b']])
assert str(tagsInConditions) == str(['#b']) assert str(tagsInConditions) == str(['#b'])
hashtags = ['#y', '#z'] hashtags = ['#y', '#z']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#a', '#b', '#c'] hashtags = ['#a', '#b', '#c']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
conditionsStr = '#foo or #bar and #a' conditionsStr = '#foo or #bar and #a'
tagsInConditions = [] tagsInConditions = []
@ -2255,11 +2271,11 @@ def testHashtagRuleTree():
assert str(tree) == str(['and', ['or', ['#foo'], ['#bar']], ['#a']]) assert str(tree) == str(['and', ['or', ['#foo'], ['#bar']], ['#a']])
assert str(tagsInConditions) == str(['#foo', '#bar', '#a']) assert str(tagsInConditions) == str(['#foo', '#bar', '#a'])
hashtags = ['#bar', '#a'] hashtags = ['#bar', '#a']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#foo', '#a'] hashtags = ['#foo', '#a']
assert hashtagRuleResolve(tree, hashtags, moderated) assert hashtagRuleResolve(tree, hashtags, moderated, content)
hashtags = ['#x', '#a'] hashtags = ['#x', '#a']
assert not hashtagRuleResolve(tree, hashtags, moderated) assert not hashtagRuleResolve(tree, hashtags, moderated, content)
def runAllTests(): def runAllTests():