Add missing encodings to TextDecoder

cmake/targets/BuildBun.cmake

@@ -518,10 +518,15 @@ set(BUN_ZIG_OUTPUT ${BUILD_PATH}/bun-zig.o)
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm|ARM|arm64|ARM64|aarch64|AARCH64")
   if(APPLE)
     set(ZIG_CPU "apple_m1")
+    set(HOMEBREW_PREFIX "/opt/homebrew")
   else()
     set(ZIG_CPU "native")
   endif()
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|X86_64|x64|X64|amd64|AMD64")
+  if(APPLE)
+    set(HOMEBREW_PREFIX "/usr/local")
+  endif()
+
   if(ENABLE_BASELINE)
     set(ZIG_CPU "nehalem")
   else()
@@ -1010,6 +1015,8 @@ include_directories(${WEBKIT_INCLUDE_PATH})
 
 if(NOT WEBKIT_LOCAL AND NOT APPLE)
   include_directories(${WEBKIT_INCLUDE_PATH}/wtf/unicode)
+elseif(WEBKIT_PREBUILT AND APPLE)
+  include_directories(${HOMEBREW_PREFIX}/opt/icu4c/include)
 endif()
 
 # --- Dependencies ---

src/bun.js/bindings/ScriptExecutionContext.cpp

@@ -7,6 +7,7 @@
 #include "_libusockets.h"
 #include "BunClientData.h"
 #include "EventLoopTask.h"
+#include "TextCodecICU.h"
 
 extern "C" void Bun__startLoop(us_loop_t* loop);
 
@@ -17,6 +18,24 @@ static std::atomic<unsigned> lastUniqueIdentifier = 0;
 WTF_MAKE_ISO_ALLOCATED_IMPL(EventLoopTask);
 WTF_MAKE_ISO_ALLOCATED_IMPL(ScriptExecutionContext);
 
+ScriptExecutionContext::ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject)
+    : m_vm(vm)
+    , m_globalObject(globalObject)
+    , m_identifier(0)
+    , m_broadcastChannelRegistry(BunBroadcastChannelRegistry::create())
+{
+    regenerateIdentifier();
+}
+
+ScriptExecutionContext::ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject, ScriptExecutionContextIdentifier identifier)
+    : m_vm(vm)
+    , m_globalObject(globalObject)
+    , m_identifier(identifier)
+    , m_broadcastChannelRegistry(BunBroadcastChannelRegistry::create())
+{
+    addToContextsMap();
+}
+
 static Lock allScriptExecutionContextsMapLock;
 static HashMap<ScriptExecutionContextIdentifier, ScriptExecutionContext*>& allScriptExecutionContextsMap() WTF_REQUIRES_LOCK(allScriptExecutionContextsMapLock)
 {
@@ -31,6 +50,15 @@ ScriptExecutionContext* ScriptExecutionContext::getScriptExecutionContext(Script
     return allScriptExecutionContextsMap().get(identifier);
 }
 
+PAL::ICUConverterWrapper& ScriptExecutionContext::cachedConverterICU()
+{
+    if (!m_cachedConverterICU) {
+        m_cachedConverterICU = makeUnique<PAL::ICUConverterWrapper>();
+    }
+
+    return *m_cachedConverterICU;
+}
+
 template<bool SSL, bool isServer>
 static void registerHTTPContextForWebSocket(ScriptExecutionContext* script, us_socket_context_t* ctx, us_loop_t* loop)
 {

src/bun.js/bindings/ScriptExecutionContext.h

@@ -14,6 +14,10 @@
 #include "CachedScript.h"
 #include <wtf/URL.h>
 
+namespace PAL {
+class ICUConverterWrapper;
+}
+
 namespace uWS {
 template<bool isServer, bool isClient, typename UserData>
 struct WebSocketContext;
@@ -37,24 +41,8 @@ class ScriptExecutionContext : public CanMakeWeakPtr<ScriptExecutionContext> {
     WTF_MAKE_ISO_ALLOCATED(ScriptExecutionContext);
 
 public:
-    ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject)
-        : m_vm(vm)
-        , m_globalObject(globalObject)
-        , m_identifier(0)
-        , m_broadcastChannelRegistry(BunBroadcastChannelRegistry::create())
-    {
-        regenerateIdentifier();
-    }
-
-    ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject, ScriptExecutionContextIdentifier identifier)
-        : m_vm(vm)
-        , m_globalObject(globalObject)
-        , m_identifier(identifier)
-        , m_broadcastChannelRegistry(BunBroadcastChannelRegistry::create())
-    {
-        addToContextsMap();
-    }
-
+    ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject);
+    ScriptExecutionContext(JSC::VM* vm, JSC::JSGlobalObject* globalObject, ScriptExecutionContextIdentifier identifier);
     ~ScriptExecutionContext();
 
     static ScriptExecutionContextIdentifier generateIdentifier();
@@ -160,6 +148,8 @@ class ScriptExecutionContext : public CanMakeWeakPtr<ScriptExecutionContext> {
 
     static ScriptExecutionContext* getMainThreadScriptExecutionContext();
 
+    PAL::ICUConverterWrapper& cachedConverterICU();
+
 private:
     JSC::VM* m_vm = nullptr;
     JSC::JSGlobalObject* m_globalObject = nullptr;
@@ -184,6 +174,8 @@ class ScriptExecutionContext : public CanMakeWeakPtr<ScriptExecutionContext> {
     us_socket_context_t* m_connected_ssl_client_websockets_ctx = nullptr;
     us_socket_context_t* m_connected_client_websockets_ctx = nullptr;
 
+    std::unique_ptr<PAL::ICUConverterWrapper> m_cachedConverterICU = { nullptr };
+
 public:
     template<bool isSSL, bool isServer>
     us_socket_context_t* connectedWebSocketContext()

src/bun.js/bindings/exports.zig

@@ -1092,3 +1092,24 @@ fn findPathInner(
     );
     return errorable.unwrap() catch null;
 }
+
+pub const WebKitTextCodec = opaque {
+    extern fn WebKitTextCodec__create(encoding_label: [*]const u8, len: usize) ?*WebKitTextCodec;
+    extern fn WebKitTextCodec__deinit(this: *WebKitTextCodec) void;
+    extern fn WebKitTextCodec__decode(this: *WebKitTextCodec, ptr: [*]const u8, len: usize, flush: bool, stopOnError: *bool) bun.String;
+    extern fn WebKitTextCodec__stripByteOrderMark(this: *WebKitTextCodec) void;
+    extern fn WebKitTextCodec__name(this: *WebKitTextCodec) bun.String;
+    pub fn init(encoding_label: []const u8) ?*WebKitTextCodec {
+        return WebKitTextCodec__create(encoding_label.ptr, encoding_label.len);
+    }
+
+    pub const name = WebKitTextCodec__name;
+
+    pub const deinit = WebKitTextCodec__deinit;
+
+    pub fn decode(this: *WebKitTextCodec, input: []const u8, flush: bool, stop_on_error: *bool) bun.String {
+        return WebKitTextCodec__decode(this, input.ptr, input.len, flush, stop_on_error);
+    }
+
+    pub const stripByteOrderMark = WebKitTextCodec__stripByteOrderMark;
+};

src/bun.js/bindings/root.h

@@ -73,6 +73,11 @@
 #define WEBCORE_EXPORT JS_EXPORT_PRIVATE
 #endif
 
+#if OS(DARWIN)
+// Prevent symbol names from causing issues
+#define U_DISABLE_RENAMING 1
+#endif
+
 #include <wtf/PlatformCallingConventions.h>
 #include <JavaScriptCore/JSCJSValue.h>
 #include <wtf/text/MakeString.h>

src/bun.js/bindings/webcore/DecodeEscapeSequences.h

@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2011 Daniel Bates ([email protected]). All Rights Reserved.
+ * Copyright (c) 2012 Google, inc.  All Rights Reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Google Inc. nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "TextEncoding.h"
+#include <wtf/ASCIICType.h>
+#include <wtf/Assertions.h>
+#include <wtf/text/StringBuilder.h>
+
+namespace PAL {
+
+// See <http://en.wikipedia.org/wiki/Percent-encoding#Non-standard_implementations>.
+struct Unicode16BitEscapeSequence {
+    enum { SequenceSize = 6 }; // e.g. %u26C4
+    static size_t findInString(StringView string, size_t startPosition) { return string.find("%u"_s, startPosition); }
+    static size_t findEndOfRun(StringView string, size_t startPosition, size_t endPosition)
+    {
+        size_t runEnd = startPosition;
+        while (endPosition - runEnd >= SequenceSize && string[runEnd] == '%' && string[runEnd + 1] == 'u'
+            && isASCIIHexDigit(string[runEnd + 2]) && isASCIIHexDigit(string[runEnd + 3])
+            && isASCIIHexDigit(string[runEnd + 4]) && isASCIIHexDigit(string[runEnd + 5])) {
+            runEnd += SequenceSize;
+        }
+        return runEnd;
+    }
+    static String decodeRun(StringView run, const TextEncoding&)
+    {
+        // Each %u-escape sequence represents a UTF-16 code unit.
+        // See <http://www.w3.org/International/iri-edit/draft-duerst-iri.html#anchor29>.
+        // For 16-bit escape sequences, we know that findEndOfRun() has given us a contiguous run of sequences
+        // without any intervening characters, so decode the run without additional checks.
+        auto numberOfSequences = run.length() / SequenceSize;
+        StringBuilder builder;
+        builder.reserveCapacity(numberOfSequences);
+        while (numberOfSequences--) {
+            UChar codeUnit = (toASCIIHexValue(run[2]) << 12) | (toASCIIHexValue(run[3]) << 8) | (toASCIIHexValue(run[4]) << 4) | toASCIIHexValue(run[5]);
+            builder.append(codeUnit);
+            run = run.substring(SequenceSize);
+        }
+        return builder.toString();
+    }
+};
+
+struct URLEscapeSequence {
+    enum { SequenceSize = 3 }; // e.g. %41
+    static size_t findInString(StringView string, size_t startPosition) { return string.find('%', startPosition); }
+    static size_t findEndOfRun(StringView string, size_t startPosition, size_t endPosition)
+    {
+        // Make the simplifying assumption that supported encodings may have up to two unescaped characters
+        // in the range 0x40 - 0x7F as the trailing bytes of their sequences which need to be passed into the
+        // decoder as part of the run. In other words, we end the run at the first value outside of the
+        // 0x40 - 0x7F range, after two values in this range, or at a %-sign that does not introduce a valid
+        // escape sequence.
+        size_t runEnd = startPosition;
+        int numberOfTrailingCharacters = 0;
+        while (runEnd < endPosition) {
+            if (string[runEnd] == '%') {
+                if (endPosition - runEnd >= SequenceSize && isASCIIHexDigit(string[runEnd + 1]) && isASCIIHexDigit(string[runEnd + 2])) {
+                    runEnd += SequenceSize;
+                    numberOfTrailingCharacters = 0;
+                } else
+                    break;
+            } else if (string[runEnd] >= 0x40 && string[runEnd] <= 0x7F && numberOfTrailingCharacters < 2) {
+                runEnd += 1;
+                numberOfTrailingCharacters += 1;
+            } else
+                break;
+        }
+        return runEnd;
+    }
+
+    static Vector<uint8_t, 512> decodeRun(StringView run)
+    {
+        // For URL escape sequences, we know that findEndOfRun() has given us a run where every %-sign introduces
+        // a valid escape sequence, but there may be characters between the sequences.
+        Vector<uint8_t, 512> buffer;
+        buffer.grow(run.length()); // Unescaping hex sequences only makes the length smaller.
+        size_t bufferIndex = 0;
+        while (!run.isEmpty()) {
+            if (run[0] == '%') {
+                buffer[bufferIndex++] = (toASCIIHexValue(run[1]) << 4) | toASCIIHexValue(run[2]);
+                run = run.substring(SequenceSize);
+            } else {
+                buffer[bufferIndex++] = run[0];
+                run = run.substring(1);
+            }
+        }
+        buffer.shrink(bufferIndex);
+        return buffer;
+    }
+
+    static String decodeRun(StringView run, const TextEncoding& encoding)
+    {
+        auto buffer = decodeRun(run);
+        if (!encoding.isValid())
+            return PAL::UTF8Encoding().decode(buffer.span());
+        return encoding.decode(buffer.span());
+    }
+};
+
+template<typename EscapeSequence>
+String decodeEscapeSequences(StringView string, const TextEncoding& encoding)
+{
+    StringBuilder result;
+    size_t length = string.length();
+    size_t decodedPosition = 0;
+    size_t searchPosition = 0;
+    size_t encodedRunPosition;
+    while ((encodedRunPosition = EscapeSequence::findInString(string, searchPosition)) != notFound) {
+        size_t encodedRunEnd = EscapeSequence::findEndOfRun(string, encodedRunPosition, length);
+        searchPosition = encodedRunEnd;
+        if (encodedRunEnd == encodedRunPosition) {
+            ++searchPosition;
+            continue;
+        }
+
+        String decoded = EscapeSequence::decodeRun(string.substring(encodedRunPosition, encodedRunEnd - encodedRunPosition), encoding);
+        if (decoded.isEmpty())
+            continue;
+
+        result.append(string.substring(decodedPosition, encodedRunPosition - decodedPosition), decoded);
+        decodedPosition = encodedRunEnd;
+    }
+    result.append(string.substring(decodedPosition, length - decodedPosition));
+    return result.toString();
+}
+
+inline Vector<uint8_t> decodeURLEscapeSequencesAsData(StringView string)
+{
+    Vector<uint8_t> result;
+    size_t decodedPosition = 0;
+    size_t searchPosition = 0;
+    while (true) {
+        size_t encodedRunPosition = URLEscapeSequence::findInString(string, searchPosition);
+        size_t encodedRunEnd = 0;
+        if (encodedRunPosition != notFound) {
+            encodedRunEnd = URLEscapeSequence::findEndOfRun(string, encodedRunPosition, string.length());
+            searchPosition = encodedRunEnd;
+            if (encodedRunEnd == encodedRunPosition) {
+                ++searchPosition;
+                continue;
+            }
+        }
+
+        // Strings are encoded as requested.
+        result.appendVector(PAL::UTF8Encoding().encodeForURLParsing(string.substring(decodedPosition, encodedRunPosition - decodedPosition)));
+
+        if (encodedRunPosition == notFound)
+            return result;
+
+        // Bytes go through as-is.
+        auto decodedEscapeSequence = URLEscapeSequence::decodeRun(string.substring(encodedRunPosition, encodedRunEnd - encodedRunPosition));
+        ASSERT(!decodedEscapeSequence.isEmpty());
+        result.appendVector(decodedEscapeSequence);
+
+        decodedPosition = encodedRunEnd;
+    }
+}
+
+} // namespace PAL