🛠️ Minor adjustement on the MD around european words (#133)

* ❇️ Adjust the MD around obvious bad EU word rendered Partially close #130 * 🔖 Bump version 2.0.8.dev1
jawah · Nov 1, 2021 · 00ffea0 · 00ffea0
1 parent 618cd8c
commit 00ffea0
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 4 deletions.
diff --git a/charset_normalizer/md.py b/charset_normalizer/md.py
@@ -293,8 +293,14 @@ def feed(self, character: str) -> None:
 
             self._character_count += buffer_length
 
-            if buffer_length >= 4 and self._buffer_accent_count / buffer_length > 0.34:
-                self._is_current_word_bad = True
+            if buffer_length >= 4:
+                if self._buffer_accent_count / buffer_length > 0.34:
+                    self._is_current_word_bad = True
+                # Word/Buffer ending with a upper case accentuated letter are so rare,
+                # that we will consider them all as suspicious. Same weight as foreign_long suspicious.
+                if is_accentuated(self._buffer[-1]) and self._buffer[-1].isupper():
+                    self._foreign_long_count += 1
+                    self._is_current_word_bad = True
             if buffer_length >= 24 and self._foreign_long_watch:
                 self._foreign_long_count += 1
                 self._is_current_word_bad = True

diff --git a/charset_normalizer/version.py b/charset_normalizer/version.py
@@ -2,5 +2,5 @@
 Expose version
 """
 
-__version__ = "2.0.8.dev0"
+__version__ = "2.0.8.dev1"
 VERSION = __version__.split(".")
diff --git a/tests/test_mess_detection.py b/tests/test_mess_detection.py
@@ -13,7 +13,7 @@
         ("ïstanbul, T■rkiye'nin en kalabal»k, iktisadi ve k■lt■rel aÓ»dan en —nemli", 0.1, 0.5),
         ("<i>Parce que Óa, c'est la vÕritable histoire de la rencontre avec votre Tante Robin.</i>", 0.01, 0.5),
         ("""ØĢØŠØģØ§ØĶŲ ŲŲ ØĢŲ Ø§ŲŲØ§Øģ ŲŲŲ ŲØ§ ØģŲŲŲØŠØģØ§ØĶŲŲŲØ ØŊØđŲØ§ ŲØģŲØđ ØđŲ (ŲØąŲØŊŲ) ŲØ§ŲØŪØ§ØŠŲ""", 0.8, 2.0),
-        ("""ÇáÚŞáíÉ , ÇáÊäæíã ÇáãÛäÇØíÓí æ / Ãæ ÇáÇŞÊÑÇÍ""", 0.8, 2.0),
+        ("""ÇáÚŞáíÉ , ÇáÊäæíã ÇáãÛäÇØíÓí æ / Ãæ ÇáÇŞÊÑÇÍ""", 0.8, 2.5),
         ("""[email protected] ุชุฑุฌูููุฉ ููุดูููุงู ุงููููููููุงูRadoZ ุชูููุนููููุฏูููู ุงููููุชูููููููููููููุช ููููู ูููุจููู""", 0.5, 2.0)
 
     ]