[WIP] isValidUtf8 function, tests

.gitignore

@@ -8,3 +8,4 @@ cabal.sandbox.config
 .hsenv
 *~
 dist-newstyle/
+.nvimrc

Changelog.md

@@ -4,6 +4,10 @@
 
 [0.12.0.0]: https://github.com/haskell/bytestring/compare/0.11.1.0...0.12.0.0
 
+[0.11.2.0] - September 2021
+
+* Add `Data.ByteString.isValidUtf8`
+
 [0.11.1.0] — February 2021
 
 * [Add `Data.ByteString.Char8.findIndexEnd` and `Data.ByteString.Lazy.Char8.{elemIndexEnd,findIndexEnd,unzip}`](https://github.com/haskell/bytestring/pull/342)

Data/ByteString.hs

@@ -1,4 +1,5 @@
 {-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE MagicHash #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE TupleSections #-}
 {-# OPTIONS_HADDOCK prune #-}
@@ -141,6 +142,9 @@ module Data.ByteString (
         isSuffixOf,
         isInfixOf,
 
+        -- ** Encoding validation
+        isValidUtf8,
+
         -- ** Search for arbitrary substrings
         breakSubstring,
 
@@ -238,7 +242,7 @@ import Control.Exception        (IOException, catch, finally, assert, throwIO)
 import Control.Monad            (when, void)
 
 import Foreign.C.String         (CString, CStringLen)
-import Foreign.C.Types          (CSize)
+import Foreign.C.Types          (CSize (CSize), CInt (CInt))
 import Foreign.ForeignPtr       (ForeignPtr, withForeignPtr, touchForeignPtr)
 import Foreign.ForeignPtr.Unsafe(unsafeForeignPtrToPtr)
 import Foreign.Marshal.Alloc    (allocaBytes)
@@ -1523,6 +1527,17 @@ stripSuffix bs1@(BS _ l1) bs2@(BS _ l2)
 isInfixOf :: ByteString -> ByteString -> Bool
 isInfixOf p s = null p || not (null $ snd $ breakSubstring p s)
 
+-- | /O(n)/ Check whether a 'ByteString' represents valid UTF-8.
+--
+-- @since 0.11.2.0
+isValidUtf8 :: ByteString -> Bool
+isValidUtf8 (BS ptr len) = accursedUnutterablePerformIO $ unsafeWithForeignPtr ptr $ \p -> do 
+  CInt i <- isValidUtf8# p (CSize . fromIntegral $ len)
+  pure $ i /= 0
+
+foreign import ccall unsafe "is_valid_utf8" isValidUtf8#
+  :: Ptr Word8 -> CSize -> IO CInt
+
 -- | Break a string on a substring, returning a pair of the part of the
 -- string prior to the match, and the rest of the string.
 --

bytestring.cabal

@@ -1,5 +1,5 @@
 Name:                bytestring
-Version:             0.11.1.0
+Version:             0.11.2.0
 Synopsis:            Fast, compact, strict and lazy byte strings with a list interface
 Description:
     An efficient compact, immutable byte string type (both strict and lazy)
@@ -113,9 +113,15 @@ library
                     -fmax-simplifier-iterations=10
                     -fdicts-cheap
                     -fspec-constr-count=6
+  if (arch(x86_64) || arch(i386))
+    c-sources:        cbits/fpstring.c
+                      cbits/itoa.c
+                      cbits/x86/is-valid-utf8.c
+  else
+    c-sources:        cbits/fpstring.c
+                      cbits/itoa.c
+                      cbits/is-valid-utf8.c
 
-  c-sources:         cbits/fpstring.c
-                     cbits/itoa.c
   cc-options:        -std=c11
   include-dirs:      include
   includes:          fpstring.h
@@ -172,6 +178,14 @@ test-suite bytestring-th
   ghc-options:      -Wall -fwarn-tabs -threaded -rtsopts
   default-language: Haskell2010
 
+test-suite is-valid-utf8
+  type:             exitcode-stdio-1.0
+  hs-source-dirs:   tests/is-valid-utf8
+  main-is:          Main.hs
+  build-depends:    base, bytestring, tasty, tasty-quickcheck, QuickCheck
+  ghc-options:      -Wall -fwarn-tabs -threaded -rtsopts
+  default-language: Haskell2010
+
 benchmark bytestring-bench
   main-is:          BenchAll.hs
   other-modules:    BenchBoundsCheckFusion

cabal.project.local

@@ -0,0 +1,2 @@
+ignore-project: False
+with-compiler: ghc-9.0.1

cbits/is-valid-utf8.c

@@ -0,0 +1,108 @@
+#pragma GCC push_options
+#pragma GCC optimize("-O2")
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+// 0x80 in every 'lane'.
+static uint64_t const high_bits_mask = 0x8080808080808080ULL;
+
+int is_valid_utf8(uint8_t const *const src, size_t const len) {
+  uint8_t const *ptr = (uint8_t const *)src;
+  // This is 'one past the end' to make loop termination and bounds checks
+  // easier.
+  uint8_t const *const end = ptr + len;
+  while (ptr < end) {
+    uint8_t const byte = *ptr;
+    // Check if the byte is ASCII.
+    if (byte <= 0x7F) {
+      ptr++;
+      // If we saw one ASCII byte, as long as it's not whitespace, it's quite
+      // likely we'll see more.
+      bool is_not_whitespace = byte > 32;
+      // If possible, do a block-check ahead.
+      if ((ptr + 32 < end) && is_not_whitespace) {
+        uint64_t const *big_ptr = (uint64_t const *)ptr;
+        // Non-ASCII bytes have a set MSB. Thus, if we AND with 0x80 in every
+        // 'lane', we will get 0 if everything is ASCII, and something else
+        // otherwise.
+        uint64_t results[4] = {(*big_ptr) & high_bits_mask,
+                               (*(big_ptr + 1)) & high_bits_mask,
+                               (*(big_ptr + 2)) & high_bits_mask,
+                               (*(big_ptr + 3)) & high_bits_mask};
+        if (results[0] == 0) {
+          ptr += 8;
+          if (results[1] == 0) {
+            ptr += 8;
+            if (results[2] == 0) {
+              ptr += 8;
+              if (results[3] == 0) {
+                ptr += 8;
+              } else {
+                ptr += (__builtin_ctzl(results[3]) / 8);
+              }
+            } else {
+              ptr += (__builtin_ctzl(results[2]) / 8);
+            }
+          } else {
+            ptr += (__builtin_ctzl(results[1]) / 8);
+          }
+        } else {
+          ptr += (__builtin_ctzl(results[0]) / 8);
+        }
+      }
+    }
+    // Check for a valid 2-byte sequence.
+    //
+    // We use a signed comparison to avoid an extra comparison with 0x80, since
+    // _signed_ 0x80 is -128.
+    else if (ptr + 1 < end && byte >= 0xC2 && byte <= 0xDF &&
+             ((int8_t) * (ptr + 1)) <= (int8_t)0xBF) {
+      ptr += 2;
+    }
+    // Check for a valid 3-byte sequence.
+    else if (ptr + 2 < end && byte >= 0xE0 && byte <= 0xEF) {
+      uint8_t const byte2 = *(ptr + 1);
+      bool byte2_valid = (int8_t)byte2 <= (int8_t)0xBF;
+      bool byte3_valid = ((int8_t) * (ptr + 2)) <= (int8_t)0xBF;
+      if (byte2_valid && byte3_valid &&
+          // E0, A0..BF, 80..BF
+          ((byte == 0xE0 && byte2 >= 0xA0) ||
+           // E1..EC, 80..BF, 80..BF
+           (byte >= 0xE1 && byte <= 0xEC) ||
+           // ED, 80..9F, 80..BF
+           (byte == 0xED && byte2 <= 0x9F) ||
+           // EE..EF, 80..BF, 80..BF
+           (byte >= 0xEE && byte <= 0xEF))) {
+        ptr += 3;
+      } else {
+        return 0;
+      }
+    }
+    // Check for a valid 4-byte sequence.
+    else if (ptr + 3 < end) {
+      uint8_t const byte2 = *(ptr + 1);
+      bool byte2_valid = (int8_t)byte2 <= (int8_t)0xBF;
+      bool byte3_valid = ((int8_t) * (ptr + 2)) <= (int8_t)0xBF;
+      bool byte4_valid = ((int8_t) * (ptr + 3)) <= (int8_t)0xBF;
+      if (byte2_valid && byte3_valid && byte4_valid &&
+          // F0, 90..BF, 80..BF, 80..BF
+          ((byte == 0xF0 && byte2 >= 0x90) ||
+           // F1..F3, 80..BF, 80..BF, 80..BF
+           (byte >= 0xF1 && byte <= 0xF3) ||
+           // F4, 80..8F, 80..BF, 80..BF
+           (byte == 0xF4 && byte2 <= 0x8F))) {
+        ptr += 4;
+      } else {
+        return 0;
+      }
+    }
+    // Otherwise, invalid.
+    else {
+      return 0;
+    }
+  }
+  // If we got this far, we're valid.
+  return 1;
+}
+#pragma GCC pop_options