Improve check_zh_en

alanshi · Sep 29, 2024 · fd4f98a · fd4f98a
1 parent ab56f7e
commit fd4f98a
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -33,6 +33,16 @@ print(f"文件名: {file_path}, 编码: {coding_name}")
 
 ```
 
+##### 检测文档中英文比例
+```
+from charset_mnbvc import api
+
+file_path = "test.txt"
+ret, percentage = api.check_zh_en(file_path)
+print(f"中英文文档: {ret}, 比例: {percentage}")
+
+```
+
 ##### 获取二进制数据编码
 ```
 from charset_mnbvc import api
@@ -207,7 +217,7 @@ python pre_check.py -i /Users/alan/temp_test/20230101/aliyun.20230101.8.武侠
 已将检测出错结果保存至 check_result_1706672423.csv 文件中,请查阅!
 ```
 
-#### 中英文混合编码检测工具
+#### 中英文混合编码检测测试范例
 ```
 usage: python examples/check_zh_en.py
 tests/fixtures/test_sample_gbk_锟斤铐.txt False zh,en percentage:82.76%

diff --git a/charset_mnbvc/api.py b/charset_mnbvc/api.py
@@ -39,7 +39,7 @@ def check_zh_en(data) -> bool:
     total_bytes = len(data)
     zh_en_count = 0
     # count the number of Chinese and English characters
-    TIPS_CONTEXT_RANGE = 98
+    TIPS_CONTEXT_RANGE = 96
     for char in data:
         if re_zh_en.match(char):
             zh_en_count += 1

diff --git a/charset_mnbvc/constant.py b/charset_mnbvc/constant.py
@@ -22,7 +22,7 @@
                     r'\u5206\u8fd8\u8fdb\u597d\u5c0f\u90e8\u5176\u4e9b\u4e3b\u6837\u7406\u5fc3\u5979' \
                     r'\u672c\u524d\u5f00\u4f46\u56e0\u53ea\u4ece\u60f3\u5b9e]'
 
-REGEX_ZH_EN = r'[\u0009\u000a\u0020-\u007f\uff01-\uff64\u2002\u2003\u2010-\u2027\u202f-\u205f\u2100-\u214f\u2200-\u22ff\u3000-\u303f\u2150-\u218b\u207f-\u208e\u2000-\u206F\u4e00-\u9fff\u0021-\u007e\u2018-\u201f\uFEFF\u00A9\u00A0\uFFFD\u00E9\uFF5E\uD83D\uFFE5\uFF0D\u00D7\u00B6\uFF5C\u00A5\u00A2\u00F6\u00E1\u00E4\u00F3\uFF0F\u00FC\u2192\u26A0\u251C\uD83C\u00ED\uD83E\uFF1D\u00E7\u200B\uFF0B\u25CF\u00C2\u00FA\u00F8\u00C4\u00AB\u00E5\u2028\u00E0\u0161\u00E8\u00AE\u00B0\u2460\u2265\u041F\u00F1\u00B1\u2217\u010D\u00EB\u017E\u221E\u0142\u00BB\u00B5\uFF3B\u00DF\u2264\u0421\u2514\uC624\u00C6\u2212\u2713\u00C1\u00CF\u00C9\u250C\u0627\u00A7\u221A\u00E3\u306E\u0410\u2193\u00EA\u2588\u0411\u00A8\u0131\u2705\u00E6\u041A\u041D\u015B\u0412\u0103\uFB01\u00B2\uFFFF\u20AC\u2190\u2191\uFF05\u0178\u2318\u2581\u0645\u03B7\u2030\u00A1\u041E\u0414\u041C\u0422\uFF03\u2118\u03BC\u0938\uB9C8\u00F4\uE86F\u0101\u200F\u0418\u062A\u00EF\u274C\u03B1\u0417\u00D6\uFE0F\u0915\u0423\u2605\u2122\u03C0\u00A3\u00D8\u2728\u00C0\u00C5\u0107\u00B4\u00E2\u200E\u00F5\u00F7\uFF11\u2260\uFF0A\u0430\u33A1\u00AC\u25CB\u2103\uFF34\u3007\u2502\u00F2\u00C7\u03A3\u0119\u021B\u0259\u2714\u266A\u25C6\u00F9\u00EE\u00DC\u2015\u0420\u2011\u0159\u05D0\u25BC\u00CD\uFF1E\u00FD\u0096\u00EC\u0130\u3053\u0110\u03C3\u21B5\u015F\u00F0\u00C3\u32A3\u2248\u0391\u0105\u0141\u09B8\u092A\u0111\u05DE\u2764\u2F64\u0113\u043F\u012B\u2500\u05D4\u0395\u0628\u203B\u2229\u21B3\u0413\u0117\u011F\u017C\u20B1\u00D3\u250F\u00A4\u0160\u00BF\u0633\u0394\u2B07\u306F\u2208\u00A6\uFB00\u03BB\u0438\u043A\uFF1C\u0416\u0905\u0424\u263A\u02C9\u0144\u3042\u0E01\uFF21\u203A\u0165\u1021\u12E8\u00CE\u00B9\u1780\u00BD\u0623\u03F5\u0085\u21D2\uC774\u092E\u0192\u03A0\uFF02\u0646\u0E40\u00AD\uFFFC\u2123\u2211\u00AF\u2F08\u016B\u05E9\u30A2\u00C8\u015E\u0995\u0441\u012C\u0089\u03B2\uD835\u2209\u062C\u25B2\u03A9\u2F0A\u201E\u043D\u06A9\u2160\u0635\u306B\u011B\u010C\u0434\u2717\u05EA\u00D0\u30E8\u05b5\u067E\u2299\uD55C\u30D5\u30B5\u0368\u017D\u00D1\u0431\u00AA\uFF10\u00B3\u0442\u00BA\u0935\u062F\u0A38\u2161\u03B5\u0151\uFF5B\u0641\u00DA\uFB02\uFF06\u09AA\u2F00\u2031\u2006\u2795\u0631\u00CA\u3092\u2603\uE629\u200A\u2B50\u25A0\u092B\uFE51\u1794\u2236\uB418\u00FF\u03B8\u0CB8\u039A\u2716\u2757\u30FB\u0BAA\u042D\u02bb\u0432\u00D2\u10E1\u304B\u039C\u25B6\uFF12\u0219\u0906\u0985\u3016\u0928\u0E81\u0D38\u256D\u2F8F\u05E2\u02D9\u043C\u0DC3\u0648\u2032\u0AB8\u0921\u09AC\u00CB\u00FB\u2162\u0427\u0644\u25A1\u0425\u041B\u0639\u1019\u0C38\u2606\u0524\u0B85\u179F\u03B4\u098F\u00B8\u0308\uFF61\u0E3F\u279C\uFB03\u0C15\u0E44\u0986\u0163\u00BC\uFF29\u2215\u30C7\u10DB\u0D15\u0642\u00D4\u276F\u03C6\uBAA8\u062E\u0E15\u0B9A\u0531\u092C\u0DB4\u0443\u0939\u0B38\u22EF\u00D5\u0625\u05F4\u1798\u03B3\u0437\u0435\u0E04\uB768\u2F40\u062D\u0B95\u0415\u02f5\u3067\u221D\u2039\u0BAE\u0173\u30E5\u0622\u0E2D\u0440\u064A\u04B5\u0540\u0A15\u02B5\u2753\u2F42\u2194\u0C2A\u0109\u0D05\u0393\u0E41\u03AA\u0BA4\u101E\uFF20\u2033\u012F\u3068\u2170\uFFE3\u0A95\u25BE\u30B3\u0C2E\uFF3F\u05D1\u05D9\u3057\u042F\u0C95\u30E1\uE511\u12A0\u2202\u1EBF\u30E6\u05DC\u2303\uF076\u2F45\u013E\u1015\u0647\u0E42\u0426\u30C6\uE600\u02DA\u30AB\u10D0\u0120\u2081\uB370\u0E9A\u0433\u2205\u2139\u03A4\u2206\u043E\u2020\u1000\u0092\u30B9\u0EC0\u0E02\u0C85\u02C6\u2EDB\u0AAA\u014D\u044F\u0E2A\u2016\u26A1\u1EA3\u1EC5\uC0AC\u0BB5\u30B7\u025B\u2501\u09AB\u201A\u30AD\uF047\u2261\u30D3\u060C\u1005\u0634\u0153\u0C05\u066A\u3086\u09AE\u0406\u0428\u03BA\u1016\u0B15\u0301\uCFA8\u00FE\u306A\u011D\u2B06\u0D2A\u2620\u0093\u30A8\u30D7\u2462\u0907\u304C\u222A\uAFCE\u00CC\uB150\u091C\u0930\u0446\u17A2\u0D35\u09A1\u0E23\u06AF\u0123\u2554\u0A2A\u0EAA\u0CAE\u0439\u039B\u2ED3\uFF15\u30EA\u2665\u26D4\u05B8\u03C1\u2461\u01B0\u256F\u30D1\u2267\u25BD\u2543\uB2E4\u0097\u0444\uCE5C\u0661\uC2DC\u0D28\u3125\u3126\u0102\u25B8\u09A8\u090F\u2611\u0C35\u039F\u22C5\u30AF\u0C28\u30E2\uFF04\u03C9\u25BA\u013C\u0924\u03A6\u3124\u0399\u27A1\u0E22\u2FA6\u05D7\u304A\u0436\u0E2B\u05E8\u21E7\u200C\uFF40\uFF71\u017A\uFE52\uE219\u0CA8\u0EC2\u0E1B\u049A\u3221\u4DAE\uC544\uFF2D\u02DC\u25CC\u0B87\uC5D0\u0B89\u1ED9\u304D\u016D\u2F63\u0CAA\uF967\u05E0\u0917\u307F\u0146\u30ED\u091F\uC548\uFF23\uE583\u0254\u30BF\u0A05\u014B\u0398\uC6B0\u0E95\u0084\u2228\u0932\u30A4\uFF65\u0544\u3021\u0AA8\u0148\u10D9\u0091\u053F\u30C9\u02C7\u1006\u02BC\u1791\u25C4\u0081\u272B\u0A85\uFE30\u0DB8\u0AAE\uD801\u03C4\u0100\uFF3D\uF06C\uB85C\u012E\u10D2\u099B\u039D\u099F\uAD1C\uFF14\u25B3\u0D2E\u0E1A\u05E7\u0CB5\u30DE\u2012\u1260\uC77C\u091A\uAC00\u0BA8\u0909\uFF3C\u25B7\u2715\uFF13\uD842\u2699\u10D8\u1EA1\u10D3\u2F83\uFF88\u0B06\uFE5D\u2219\u3055\u013F\u015A\u0532\u0660\u00BE\u2287\u30AA\u0643\u0D85\uC124\uFF4F\uBE44\u20A9\u04B3\uAE30\u200D\u0967\u0936\u0987\u23F3\u2565\u0A24\u0408\uFF5D\u2F46\u1014\u0EC1\u0CB9\u2009\u0445\u2025\u0D2B\u03C7\u022B\u0EAD\u0AAB\uD800\u1001\uF8FF\u223C\u252C\u21A9\u05D3\u0926\u261E\u301C\uFF41\uC804\u03BF\u10D4\u30C4\u101B\u2200\u3044\u0D9A\u0397\u0DC0\u179A\uBE14\u009C\u0997\u10E8\u25E6\u0082\u015D\u0B2A\uD464\uFF45\u2EDA\u03A1\u20B9\u04E8\u30BD\u2F47\u0CAB\u0E82\u008B\u04AE\uB0B4\u2116\u0DAF\u0B05\u0108\u037C\u3127\u25C9\u0546\u0087\u0C97\uff0c\u3002\u300a\u300b\u3001\uff1f\u2019\uff1b\u201d\uff1a\u201c\u005b\u005d\u00b7\u0060\u0021\u0040\u0023\u0024\u0025\u005e\u0026\u002a\u0028\u0029\u005f\u002b\u003d\u002d\u007c]'
+REGEX_ZH_EN = r'[\u0020\uf8f5\u000a\u000d\u000c\u0009\u0000‐\u001F\u007f\u00a9\u0009\u000a\u0020-\u007f\uff01-\uff64\u2002\u2003\u2010-\u2027\u202f-\u205f\u2100-\u214f\u2200-\u22ff\u3000-\u303f\u2150-\u218b\u207f-\u208e\u2000-\u206F\u4e00-\u9fff\u0021-\u007e\u2018-\u201f\uFEFF\u00A9\u00A0\uFFFD\u00E9\uFF5E\uD83D\uFFE5\uFF0D\u00D7\u00B6\uFF5C\u00A5\u00A2\u00F6\u00E1\u00E4\u00F3\uFF0F\u00FC\u2192\u26A0\u251C\uD83C\u00ED\uD83E\uFF1D\u00E7\u200B\uFF0B\u25CF\u00C2\u00FA\u00F8\u00C4\u00AB\u00E5\u2028\u00E0\u0161\u00E8\u00AE\u00B0\u2460\u2265\u041F\u00F1\u00B1\u2217\u010D\u00EB\u017E\u221E\u0142\u00BB\u00B5\uFF3B\u00DF\u2264\u0421\u2514\uC624\u00C6\u2212\u2713\u00C1\u00CF\u00C9\u250C\u0627\u00A7\u221A\u00E3\u306E\u0410\u2193\u00EA\u2588\u0411\u00A8\u0131\u2705\u00E6\u041A\u041D\u015B\u0412\u0103\uFB01\u00B2\uFFFF\u20AC\u2190\u2191\uFF05\u0178\u2318\u2581\u0645\u03B7\u2030\u00A1\u041E\u0414\u041C\u0422\uFF03\u2118\u03BC\u0938\uB9C8\u00F4\uE86F\u0101\u200F\u0418\u062A\u00EF\u274C\u03B1\u0417\u00D6\uFE0F\u0915\u0423\u2605\u2122\u03C0\u00A3\u00D8\u2728\u00C0\u00C5\u0107\u00B4\u00E2\u200E\u00F5\u00F7\uFF11\u2260\uFF0A\u0430\u33A1\u00AC\u25CB\u2103\uFF34\u3007\u2502\u00F2\u00C7\u03A3\u0119\u021B\u0259\u2714\u266A\u25C6\u00F9\u00EE\u00DC\u2015\u0420\u2011\u0159\u05D0\u25BC\u00CD\uFF1E\u00FD\u0096\u00EC\u0130\u3053\u0110\u03C3\u21B5\u015F\u00F0\u00C3\u32A3\u2248\u0391\u0105\u0141\u09B8\u092A\u0111\u05DE\u2764\u2F64\u0113\u043F\u012B\u2500\u05D4\u0395\u0628\u203B\u2229\u21B3\u0413\u0117\u011F\u017C\u20B1\u00D3\u250F\u00A4\u0160\u00BF\u0633\u0394\u2B07\u306F\u2208\u00A6\uFB00\u03BB\u0438\u043A\uFF1C\u0416\u0905\u0424\u263A\u02C9\u0144\u3042\u0E01\uFF21\u203A\u0165\u1021\u12E8\u00CE\u00B9\u1780\u00BD\u0623\u03F5\u0085\u21D2\uC774\u092E\u0192\u03A0\uFF02\u0646\u0E40\u00AD\uFFFC\u2123\u2211\u00AF\u2F08\u016B\u05E9\u30A2\u00C8\u015E\u0995\u0441\u012C\u0089\u03B2\uD835\u2209\u062C\u25B2\u03A9\u2F0A\u201E\u043D\u06A9\u2160\u0635\u306B\u011B\u010C\u0434\u2717\u05EA\u00D0\u30E8\u05b5\u067E\u2299\uD55C\u30D5\u30B5\u0368\u017D\u00D1\u0431\u00AA\uFF10\u00B3\u0442\u00BA\u0935\u062F\u0A38\u2161\u03B5\u0151\uFF5B\u0641\u00DA\uFB02\uFF06\u09AA\u2F00\u2031\u2006\u2795\u0631\u00CA\u3092\u2603\uE629\u200A\u2B50\u25A0\u092B\uFE51\u1794\u2236\uB418\u00FF\u03B8\u0CB8\u039A\u2716\u2757\u30FB\u0BAA\u042D\u02bb\u0432\u00D2\u10E1\u304B\u039C\u25B6\uFF12\u0219\u0906\u0985\u3016\u0928\u0E81\u0D38\u256D\u2F8F\u05E2\u02D9\u043C\u0DC3\u0648\u2032\u0AB8\u0921\u09AC\u00CB\u00FB\u2162\u0427\u0644\u25A1\u0425\u041B\u0639\u1019\u0C38\u2606\u0524\u0B85\u179F\u03B4\u098F\u00B8\u0308\uFF61\u0E3F\u279C\uFB03\u0C15\u0E44\u0986\u0163\u00BC\uFF29\u2215\u30C7\u10DB\u0D15\u0642\u00D4\u276F\u03C6\uBAA8\u062E\u0E15\u0B9A\u0531\u092C\u0DB4\u0443\u0939\u0B38\u22EF\u00D5\u0625\u05F4\u1798\u03B3\u0437\u0435\u0E04\uB768\u2F40\u062D\u0B95\u0415\u02f5\u3067\u221D\u2039\u0BAE\u0173\u30E5\u0622\u0E2D\u0440\u064A\u04B5\u0540\u0A15\u02B5\u2753\u2F42\u2194\u0C2A\u0109\u0D05\u0393\u0E41\u03AA\u0BA4\u101E\uFF20\u2033\u012F\u3068\u2170\uFFE3\u0A95\u25BE\u30B3\u0C2E\uFF3F\u05D1\u05D9\u3057\u042F\u0C95\u30E1\uE511\u12A0\u2202\u1EBF\u30E6\u05DC\u2303\uF076\u2F45\u013E\u1015\u0647\u0E42\u0426\u30C6\uE600\u02DA\u30AB\u10D0\u0120\u2081\uB370\u0E9A\u0433\u2205\u2139\u03A4\u2206\u043E\u2020\u1000\u0092\u30B9\u0EC0\u0E02\u0C85\u02C6\u2EDB\u0AAA\u014D\u044F\u0E2A\u2016\u26A1\u1EA3\u1EC5\uC0AC\u0BB5\u30B7\u025B\u2501\u09AB\u201A\u30AD\uF047\u2261\u30D3\u060C\u1005\u0634\u0153\u0C05\u066A\u3086\u09AE\u0406\u0428\u03BA\u1016\u0B15\u0301\uCFA8\u00FE\u306A\u011D\u2B06\u0D2A\u2620\u0093\u30A8\u30D7\u2462\u0907\u304C\u222A\uAFCE\u00CC\uB150\u091C\u0930\u0446\u17A2\u0D35\u09A1\u0E23\u06AF\u0123\u2554\u0A2A\u0EAA\u0CAE\u0439\u039B\u2ED3\uFF15\u30EA\u2665\u26D4\u05B8\u03C1\u2461\u01B0\u256F\u30D1\u2267\u25BD\u2543\uB2E4\u0097\u0444\uCE5C\u0661\uC2DC\u0D28\u3125\u3126\u0102\u25B8\u09A8\u090F\u2611\u0C35\u039F\u22C5\u30AF\u0C28\u30E2\uFF04\u03C9\u25BA\u013C\u0924\u03A6\u3124\u0399\u27A1\u0E22\u2FA6\u05D7\u304A\u0436\u0E2B\u05E8\u21E7\u200C\uFF40\uFF71\u017A\uFE52\uE219\u0CA8\u0EC2\u0E1B\u049A\u3221\u4DAE\uC544\uFF2D\u02DC\u25CC\u0B87\uC5D0\u0B89\u1ED9\u304D\u016D\u2F63\u0CAA\uF967\u05E0\u0917\u307F\u0146\u30ED\u091F\uC548\uFF23\uE583\u0254\u30BF\u0A05\u014B\u0398\uC6B0\u0E95\u0084\u2228\u0932\u30A4\uFF65\u0544\u3021\u0AA8\u0148\u10D9\u0091\u053F\u30C9\u02C7\u1006\u02BC\u1791\u25C4\u0081\u272B\u0A85\uFE30\u0DB8\u0AAE\uD801\u03C4\u0100\uFF3D\uF06C\uB85C\u012E\u10D2\u099B\u039D\u099F\uAD1C\uFF14\u25B3\u0D2E\u0E1A\u05E7\u0CB5\u30DE\u2012\u1260\uC77C\u091A\uAC00\u0BA8\u0909\uFF3C\u25B7\u2715\uFF13\uD842\u2699\u10D8\u1EA1\u10D3\u2F83\uFF88\u0B06\uFE5D\u2219\u3055\u013F\u015A\u0532\u0660\u00BE\u2287\u30AA\u0643\u0D85\uC124\uFF4F\uBE44\u20A9\u04B3\uAE30\u200D\u0967\u0936\u0987\u23F3\u2565\u0A24\u0408\uFF5D\u2F46\u1014\u0EC1\u0CB9\u2009\u0445\u2025\u0D2B\u03C7\u022B\u0EAD\u0AAB\uD800\u1001\uF8FF\u223C\u252C\u21A9\u05D3\u0926\u261E\u301C\uFF41\uC804\u03BF\u10D4\u30C4\u101B\u2200\u3044\u0D9A\u0397\u0DC0\u179A\uBE14\u009C\u0997\u10E8\u25E6\u0082\u015D\u0B2A\uD464\uFF45\u2EDA\u03A1\u20B9\u04E8\u30BD\u2F47\u0CAB\u0E82\u008B\u04AE\uB0B4\u2116\u0DAF\u0B05\u0108\u037C\u3127\u25C9\u0546\u0087\u0C97\uff0c\u3002\u300a\u300b\u3001\uff1f\u2019\uff1b\u201d\uff1a\u201c\u005b\u005d\u00b7\u0060\u0021\u0040\u0023\u0024\u0025\u005e\u0026\u002a\u0028\u0029\u005f\u002b\u003d\u002d\u007c]'
 
 ENCODINGS = [
     'utf_8',

diff --git a/examples/check_zh_en.py b/examples/check_zh_en.py
@@ -5,12 +5,26 @@
 
 from charset_mnbvc import api, common_utils
 
-# load directory
-folder_path = "tests/fixtures/"
 
-file_paths = common_utils.get_file_paths(folder_path)
-for file_path in file_paths:
+
+def check_zh_en_by_folder(folder_path):
+    file_paths = common_utils.get_file_paths(folder_path)
+    for file_path in file_paths:
+        with open(file_path, "rb") as f:
+            data = f.read()
+            ret, percentage = api.check_zh_en(data)
+            print(file_path,ret, f"zh,en percentage:{percentage:.2f}%")
+
+def check_zh_en_by_file(file_path):
     with open(file_path, "rb") as f:
         data = f.read()
         ret, percentage = api.check_zh_en(data)
-        print(file_path,ret, f"zh,en percentage:{percentage:.2f}%")
+        print(f"是否为中英文文档: {ret}, 比例: {percentage}")
+
+
+if __name__ == "__main__":
+    # load directory
+    folder_path = "tests/fixtures/"
+    file_path = "tests/fixtures/18.txt"
+    check_zh_en_by_folder(folder_path)
+    check_zh_en_by_file(file_path)
diff --git a/tests/fixtures/test.txt b/tests/fixtures/test.txt
@@ -1 +1 @@
-sadfkjsdflkddfsafds。d啊师傅的时代ssalkdfj史蒂夫，
+sadfkjsdflkddfsafds。d啊师傅的时代ssalkdfj史蒂夫，2321321.。。。。。
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		sadfkjsdflkddfsafds。d啊师傅的时代ssalkdfj史蒂夫，
		sadfkjsdflkddfsafds。d啊师傅的时代ssalkdfj史蒂夫，2321321.。。。。。