#1020 handle null bytes when parsing utf8 (#1021)

* add regression test for conversion of null byte * handle null bytes like single byte characters
microsoft · Jan 16, 2019 · c5ebc28 · c5ebc28
1 parent be5d43f
commit c5ebc28
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/Release/src/utilities/asyncrt_utils.cpp b/Release/src/utilities/asyncrt_utils.cpp
@@ -341,11 +341,11 @@ inline size_t count_utf8_to_utf16(const std::string& s)
 
     for (size_t index = 0; index < sSize;)
     {
-        if (sData[index] > 0)
+        if (sData[index] >= 0)
         {
             // use fast inner loop to skip single byte code points (which are
             // expected to be the most frequent)
-            while ((++index < sSize) && (sData[index] > 0))
+            while ((++index < sSize) && (sData[index] >= 0))
                 ;
 
             if (index >= sSize) break;

diff --git a/Release/tests/functional/utils/strings.cpp b/Release/tests/functional/utils/strings.cpp
@@ -153,6 +153,14 @@ SUITE(strings)
         auto result = utility::conversions::utf8_to_utf16(input);
         VERIFY_ARE_EQUAL(0x7F, result[0]);
 
+        // null byte
+        input.clear();
+        input.push_back(0);
+        input.push_back(0);
+        result = utility::conversions::utf8_to_utf16(input);
+        VERIFY_ARE_EQUAL(0, result[0]);
+        VERIFY_ARE_EQUAL(0, result[1]);
+
         // 2 byte character
         input.clear();
         // U+80