From 0e69a6278531cd410d1f76b3b7e8a16b6f53b059 Mon Sep 17 00:00:00 2001
From: Bob Mottram <bob@libreserver.org>
Date: Sat, 9 Jul 2022 11:37:33 +0100
Subject: [PATCH] Standardize text prior to filtering

---
 filters.py | 41 +++++++++++++++++++++++++++++++++++++++++
 tests.py   | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)

diff --git a/filters.py b/filters.py
index 329528ac6..318b179be 100644
--- a/filters.py
+++ b/filters.py
@@ -13,6 +13,44 @@ from utils import text_in_file
 from utils import remove_eol
 
 
+def _standardize_text_range(text: str,
+                            range_start: int, range_end: int,
+                            offset: str) -> str:
+    """Convert any fancy characters within the given range into ordinary ones
+    """
+    offset = ord(offset)
+    ctr = 0
+    text = list(text)
+    while ctr < len(text):
+        val = ord(text[ctr])
+        if val in range(range_start, range_end):
+            text[ctr] = chr(val - range_start + offset)
+        ctr += 1
+    return "".join(text)
+
+
+def standardize_text(text: str) -> str:
+    """Converts fancy unicode text to ordinary letters
+    """
+    fancy_ranges = (
+        119990, 120094, 120198, 120042, 119990, 120146, 119886
+    )
+
+    for range_start in fancy_ranges:
+        range_end = range_start + 26
+        text = _standardize_text_range(text, range_start, range_end, 'a')
+
+        range_start = range_end
+        range_end = range_start + 26
+        text = _standardize_text_range(text, range_start, range_end, 'A')
+
+    text = _standardize_text_range(text, 65345, 65345 + 26, 'a')
+    text = _standardize_text_range(text, 65313, 65313 + 26, 'A')
+    text = _standardize_text_range(text, 119964, 119964 + 26, 'A')
+
+    return text
+
+
 def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool:
     """Adds a filter for particular words within the content of a incoming posts
     """
@@ -120,6 +158,9 @@ def _is_filtered_base(filename: str, content: str) -> bool:
     if not os.path.isfile(filename):
         return False
 
+    # convert any fancy characters to ordinary ones
+    content = standardize_text(content)
+
     try:
         with open(filename, 'r', encoding='utf-8') as fp_filt:
             for line in fp_filt:
diff --git a/tests.py b/tests.py
index 990299132..a466b6b6b 100644
--- a/tests.py
+++ b/tests.py
@@ -189,6 +189,7 @@ from blocking import add_cw_from_lists
 from happening import dav_month_via_server
 from happening import dav_day_via_server
 from webapp_theme_designer import color_contrast
+from filters import standardize_text
 
 
 TEST_SERVER_GROUP_RUNNING = False
@@ -7351,6 +7352,52 @@ def _test_dogwhistles():
     assert result['hamstered']['category'] == "hamsterism"
 
 
+def _test_text_standardize():
+    print('text_standardize')
+    expected = 'This is a test'
+
+    result = standardize_text(expected)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = '𝔗𝔥𝔦𝔰 𝔦𝔰 𝔞 𝔱𝔢𝔰𝔱'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = '𝕿𝖍𝖎𝖘 𝖎𝖘 𝖆 𝖙𝖊𝖘𝖙'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = '𝓣𝓱𝓲𝓼 𝓲𝓼 𝓪 𝓽𝓮𝓼𝓽'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = '𝒯𝒽𝒾𝓈 𝒾𝓈 𝒶 𝓉𝑒𝓈𝓉'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = '𝕋𝕙𝕚𝕤 𝕚𝕤 𝕒 𝕥𝕖𝕤𝕥'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+    text = 'Ｔｈｉｓ ｉｓ ａ ｔｅｓｔ'
+    result = standardize_text(text)
+    if result != expected:
+        print(result)
+    assert result == expected
+
+
 def run_all_tests():
     base_dir = os.getcwd()
     print('Running tests...')
@@ -7368,6 +7415,7 @@ def run_all_tests():
     _test_checkbox_names()
     _test_thread_functions()
     _test_functions()
+    _test_text_standardize()
     _test_dogwhistles()
     _test_remove_end_of_line()
     _test_translation_labels()