Add setting reliableHeadRequests

lib/include/duckdb/web/config.h

@@ -70,6 +70,7 @@ struct DuckDBConfigOptions {
 struct FileSystemConfig {
     /// Allow falling back to full HTTP reads if the server does not support range requests
     std::optional<bool> allow_full_http_reads = std::nullopt;
+    std::optional<bool> reliableHeadRequests = std::nullopt;
 };
 
 struct WebDBConfig {
@@ -87,7 +88,7 @@ struct WebDBConfig {
         .cast_decimal_to_double = std::nullopt,
     };
     /// The filesystem
-    FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt};
+    FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt, .reliableHeadRequests = std::nullopt};
 
     /// These options are fetched from DuckDB
     DuckDBConfigOptions duckdb_config_options = {

lib/src/config.cc

@@ -49,6 +49,7 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
                               .filesystem =
                                   FileSystemConfig{
                                       .allow_full_http_reads = std::nullopt,
+                                      .reliableHeadRequests = std::nullopt,
                                   },
                               .duckdb_config_options =
                                   DuckDBConfigOptions{
@@ -97,6 +98,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
             if (fs.HasMember("allowFullHTTPReads") && fs["allowFullHTTPReads"].IsBool()) {
                 config.filesystem.allow_full_http_reads = fs["allowFullHTTPReads"].GetBool();
             }
+            if (fs.HasMember("reliableHeadRequests") && fs["reliableHeadRequests"].IsBool()) {
+                config.filesystem.reliableHeadRequests = fs["reliableHeadRequests"].GetBool();
+            }
         }
     }
     if (!config.query.cast_bigint_to_double.has_value()) {

lib/src/io/web_filesystem.cc

@@ -319,6 +319,10 @@ rapidjson::Value WebFileSystem::WebFile::WriteInfo(rapidjson::Document &doc) con
         filesystem_.config_->filesystem.allow_full_http_reads.value_or(true)) {
         value.AddMember("allowFullHttpReads", true, allocator);
     }
+    if ((data_protocol_ == DataProtocol::HTTP || data_protocol_ == DataProtocol::S3) &&
+        filesystem_.config_->filesystem.reliableHeadRequests.value_or(true)) {
+        value.AddMember("reliableHeadRequests", true, allocator);
+    }
     value.AddMember("collectStatistics", filesystem_.file_statistics_->TracksFile(file_name_), doc.GetAllocator());
 
     if (data_protocol_ == DataProtocol::S3) {
@@ -493,6 +497,9 @@ rapidjson::Value WebFileSystem::WriteGlobalFileInfo(rapidjson::Document &doc, ui
     if (config_->filesystem.allow_full_http_reads.value_or(true)) {
         value.AddMember("allowFullHttpReads", true, allocator);
     }
+    if (config_->filesystem.reliableHeadRequests.value_or(true)) {
+        value.AddMember("reliableHeadRequests", true, allocator);
+    }
 
     value.AddMember("s3Config", writeS3Config(config_->duckdb_config_options, allocator), allocator);
 

packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs

@@ -22,6 +22,8 @@ pub struct WebFile {
     pub data_url: Option<String>,
     #[serde(rename = "dataNativeFd")]
     pub data_native_fd: Option<u64>,
+    #[serde(rename = "reliableHeadRequests")]
+    pub allow_full_http_reads: Option<bool>,
     #[serde(rename = "allowFullHttpReads")]
     pub allow_full_http_reads: Option<bool>,
     #[serde(rename = "collectStatistics")]

packages/duckdb-wasm/src/bindings/config.ts

@@ -25,6 +25,7 @@ export interface DuckDBFilesystemConfig {
     /**
      * Allow falling back to full HTTP reads if the server does not support range requests.
      */
+    reliableHeadRequests?: boolean;
     allowFullHTTPReads?: boolean;
 }
 

packages/duckdb-wasm/src/bindings/runtime.ts

@@ -76,13 +76,15 @@ export interface DuckDBFileInfo {
     fileName: string;
     dataProtocol: DuckDBDataProtocol;
     dataUrl: string | null;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
     s3Config?: S3Config;
 }
 
 /** Global info for all files registered with DuckDB */
 export interface DuckDBGlobalFileInfo {
     cacheEpoch: number;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
     s3Config?: S3Config;
 }

packages/duckdb-wasm/src/bindings/runtime_browser.ts

@@ -159,7 +159,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                     // Supports ranges?
                     let contentLength = null;
                     let error: any | null = null;
-                    if (!file.allowFullHttpReads) {
+                    if (file.reliableHeadRequests || !file.allowFullHttpReads) {
                     try {
                         // Send a dummy HEAD request with range protocol
                         //          -> good IFF status is 206 and contentLenght is present
@@ -211,7 +211,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                             let presumedLength = null;
                             if (contentRange !== undefined) {
                                 presumedLength = contentRange;
-                            } else {
+                            } else if (!file.reliableHeadRequests) {
                                 // Send a dummy HEAD request with range protocol
                                 //          -> good IFF status is 206 and contentLenght is present
                                 const head = new XMLHttpRequest();

packages/duckdb-wasm/src/bindings/web_file.ts

@@ -8,5 +8,6 @@ export interface WebFile {
     dataUrl?: string;
     dataNativeFd?: number;
     collectStatistics?: boolean;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
 }