From 7511af13d0d123cd0cda66ce956485ecd35b7095 Mon Sep 17 00:00:00 2001 From: Bob Mottram Date: Sat, 10 Jul 2021 10:38:59 +0100 Subject: [PATCH] Limit the number of times that the same word can be repeated --- content.py | 33 +++++++++++++++++++++++++++++++++ posts.py | 4 ++++ tests.py | 39 +++++++++++++++++++++++++++++++++++++++ webapp_column_right.py | 4 ++++ webapp_post.py | 2 ++ 5 files changed, 82 insertions(+) diff --git a/content.py b/content.py index 960d3f033..253740358 100644 --- a/content.py +++ b/content.py @@ -866,6 +866,7 @@ def addHtmlTags(baseDir: str, httpPrefix: str, content = addWebLinks(content) if longWordsList: content = removeLongWords(content, maxWordLength, longWordsList) + content = limitRepeatedWords(content, 6) content = content.replace(' --linebreak-- ', '

') content = htmlReplaceEmailQuote(content) return '

' + htmlReplaceQuoteMarks(content) + '

' @@ -1053,3 +1054,35 @@ def extractTextFieldsInPOST(postBytes, boundary: str, debug: bool, postValue += postLines[line] fields[postKey] = urllib.parse.unquote(postValue) return fields + + +def limitRepeatedWords(text: str, maxRepeats: int) -> str: + """Removes words which are repeated many times + """ + words = text.replace('\n', ' ').split(' ') + repeatCtr = 0 + repeatedText = '' + replacements = {} + prevWord = '' + for word in words: + if word == prevWord: + repeatCtr += 1 + if repeatedText: + repeatedText += ' ' + word + else: + repeatedText = word + ' ' + word + else: + if repeatCtr > maxRepeats: + newText = ((prevWord + ' ') * maxRepeats).strip() + replacements[prevWord] = [repeatedText, newText] + repeatCtr = 0 + repeatedText = '' + prevWord = word + + if repeatCtr > maxRepeats: + newText = ((prevWord + ' ') * maxRepeats).strip() + replacements[prevWord] = [repeatedText, newText] + + for word, item in replacements.items(): + text = text.replace(item[0], item[1]) + return text diff --git a/posts.py b/posts.py index 0dc9e7179..07cef6cab 100644 --- a/posts.py +++ b/posts.py @@ -61,6 +61,7 @@ from utils import removeHtml from utils import dangerousMarkup from media import attachMedia from media import replaceYouTube +from content import limitRepeatedWords from content import tagExists from content import removeLongWords from content import addHtmlTags @@ -4031,6 +4032,9 @@ def downloadAnnounce(session, baseDir: str, httpPrefix: str, # remove any long words contentStr = removeLongWords(contentStr, 40, []) + # Prevent the same word from being repeated many times + contentStr = limitRepeatedWords(contentStr, 6) + # remove text formatting, such as bold/italics contentStr = removeTextFormatting(contentStr) diff --git a/tests.py b/tests.py index cbcec1c6c..eacaa47bd 100644 --- a/tests.py +++ b/tests.py @@ -94,6 +94,7 @@ from inbox import jsonPostAllowsComments from inbox import validInbox from inbox import validInboxFilenames from categories import guessHashtagCategory +from content import limitRepeatedWords from content import switchWords from content import extractTextFieldsInPOST from content import validHashTag @@ -4154,9 +4155,47 @@ def _testLimitWordLengths() -> None: assert result == "This is an exceptionally test" +def _testLimitRepetedWords() -> None: + print('limitRepeatedWords') + text = \ + "This is a preamble.\n\n" + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same\n\n" + \ + "Some other text." + expected = \ + "This is a preamble.\n\n" + \ + "Same Same Same Same Same Same\n\n" + \ + "Some other text." + result = limitRepeatedWords(text, 6) + assert result == expected + + text = \ + "This is other preamble.\n\n" + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same " + \ + "Same Same Same Same Same Same Same Same Same Same" + expected = \ + "This is other preamble.\n\n" + \ + "Same Same Same Same Same Same" + result = limitRepeatedWords(text, 6) + assert result == expected + + def runAllTests(): print('Running tests...') updateDefaultThemesList(os.getcwd()) + _testLimitRepetedWords() _testLimitWordLengths() _testSwitchWords() _testFunctions() diff --git a/webapp_column_right.py b/webapp_column_right.py index 731ac86bf..ab3cff465 100644 --- a/webapp_column_right.py +++ b/webapp_column_right.py @@ -10,6 +10,7 @@ __module_group__ = "Web Interface Columns" import os from datetime import datetime from content import removeLongWords +from content import limitRepeatedWords from utils import removeHtml from utils import locatePost from utils import loadJson @@ -265,6 +266,7 @@ def _htmlNewswire(baseDir: str, newswire: {}, nickname: str, moderator: bool, _votesIndicator(totalVotes, positiveVoting) title = removeLongWords(item[0], 16, []).replace('\n', '
') + title = limitRepeatedWords(title, 6) htmlStr += '

' + \ '' + \ @@ -293,6 +295,7 @@ def _htmlNewswire(baseDir: str, newswire: {}, nickname: str, moderator: bool, _votesIndicator(totalVotes, positiveVoting) title = removeLongWords(item[0], 16, []).replace('\n', '
') + title = limitRepeatedWords(title, 6) if moderator and moderatedItem: htmlStr += '

' + \ '') + title = limitRepeatedWords(title, 6) link = item[1] citationValue = \ diff --git a/webapp_post.py b/webapp_post.py index 3d3ff9941..a787535be 100644 --- a/webapp_post.py +++ b/webapp_post.py @@ -45,6 +45,7 @@ from utils import removeIdEnding from utils import getNicknameFromActor from utils import getDomainFromActor from utils import isEventPost +from content import limitRepeatedWords from content import replaceEmojiFromTags from content import htmlReplaceQuoteMarks from content import htmlReplaceEmailQuote @@ -1601,6 +1602,7 @@ def individualPostAsHtml(allowDownloads: bool, objectContent = \ removeLongWords(postJsonObject['object']['content'], 40, []) objectContent = removeTextFormatting(objectContent) + objectContent = limitRepeatedWords(objectContent, 6) objectContent = \ switchWords(baseDir, nickname, domain, objectContent) objectContent = htmlReplaceEmailQuote(objectContent)