Allow to disable (or postpone) some HEAD requests to allow better testing

lib/include/duckdb/web/config.h

@@ -70,6 +70,7 @@ struct DuckDBConfigOptions {
 struct FileSystemConfig {
     /// Allow falling back to full HTTP reads if the server does not support range requests
     std::optional<bool> allow_full_http_reads = std::nullopt;
+    std::optional<bool> reliable_head_requests = std::nullopt;
 };
 
 struct WebDBConfig {
@@ -87,7 +88,7 @@ struct WebDBConfig {
         .cast_decimal_to_double = std::nullopt,
     };
     /// The filesystem
-    FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt};
+    FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt, .reliable_head_requests = std::nullopt};
 
     /// These options are fetched from DuckDB
     DuckDBConfigOptions duckdb_config_options = {

lib/src/config.cc

@@ -49,6 +49,7 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
                               .filesystem =
                                   FileSystemConfig{
                                       .allow_full_http_reads = std::nullopt,
+                                      .reliable_head_requests = std::nullopt,
                                   },
                               .duckdb_config_options =
                                   DuckDBConfigOptions{
@@ -97,6 +98,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
             if (fs.HasMember("allowFullHTTPReads") && fs["allowFullHTTPReads"].IsBool()) {
                 config.filesystem.allow_full_http_reads = fs["allowFullHTTPReads"].GetBool();
             }
+            if (fs.HasMember("reliableHeadRequests") && fs["reliableHeadRequests"].IsBool()) {
+                config.filesystem.reliable_head_requests = fs["reliableHeadRequests"].GetBool();
+            }
         }
     }
     if (!config.query.cast_bigint_to_double.has_value()) {

lib/src/io/web_filesystem.cc

@@ -319,6 +319,10 @@ rapidjson::Value WebFileSystem::WebFile::WriteInfo(rapidjson::Document &doc) con
         filesystem_.config_->filesystem.allow_full_http_reads.value_or(true)) {
         value.AddMember("allowFullHttpReads", true, allocator);
     }
+    if ((data_protocol_ == DataProtocol::HTTP || data_protocol_ == DataProtocol::S3) &&
+        filesystem_.config_->filesystem.reliable_head_requests.value_or(true)) {
+        value.AddMember("reliableHeadRequests", true, allocator);
+    }
     value.AddMember("collectStatistics", filesystem_.file_statistics_->TracksFile(file_name_), doc.GetAllocator());
 
     if (data_protocol_ == DataProtocol::S3) {
@@ -493,6 +497,9 @@ rapidjson::Value WebFileSystem::WriteGlobalFileInfo(rapidjson::Document &doc, ui
     if (config_->filesystem.allow_full_http_reads.value_or(true)) {
         value.AddMember("allowFullHttpReads", true, allocator);
     }
+    if (config_->filesystem.reliable_head_requests.value_or(true)) {
+        value.AddMember("reliableHeadRequests", true, allocator);
+    }
 
     value.AddMember("s3Config", writeS3Config(config_->duckdb_config_options, allocator), allocator);
 

packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs

@@ -22,6 +22,8 @@ pub struct WebFile {
     pub data_url: Option<String>,
     #[serde(rename = "dataNativeFd")]
     pub data_native_fd: Option<u64>,
+    #[serde(rename = "reliableHeadRequests")]
+    pub reliable_head_requests: Option<bool>,
     #[serde(rename = "allowFullHttpReads")]
     pub allow_full_http_reads: Option<bool>,
     #[serde(rename = "collectStatistics")]

packages/duckdb-wasm/src/bindings/config.ts

@@ -25,6 +25,7 @@ export interface DuckDBFilesystemConfig {
     /**
      * Allow falling back to full HTTP reads if the server does not support range requests.
      */
+    reliableHeadRequests?: boolean;
     allowFullHTTPReads?: boolean;
 }
 

packages/duckdb-wasm/src/bindings/runtime.ts

@@ -76,13 +76,15 @@ export interface DuckDBFileInfo {
     fileName: string;
     dataProtocol: DuckDBDataProtocol;
     dataUrl: string | null;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
     s3Config?: S3Config;
 }
 
 /** Global info for all files registered with DuckDB */
 export interface DuckDBGlobalFileInfo {
     cacheEpoch: number;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
     s3Config?: S3Config;
 }

packages/duckdb-wasm/src/bindings/runtime_browser.ts

@@ -159,6 +159,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                     // Supports ranges?
                     let contentLength = null;
                     let error: any | null = null;
+                    if (file.reliableHeadRequests || !file.allowFullHttpReads) {
                     try {
                         // Send a dummy HEAD request with range protocol
                         //          -> good IFF status is 206 and contentLenght is present
@@ -185,6 +186,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                         error = e;
                         console.warn(`HEAD request with range header failed: ${e}`);
                     }
+                    }
 
                     // Try to fallback to full read?
                     if (file.allowFullHttpReads) {
@@ -209,8 +211,24 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                             let presumedLength = null;
                             if (contentRange !== undefined) {
                                 presumedLength = contentRange;
-                            } else if (contentLength !== null && +contentLength > 1) {
-                                presumedLength = contentLength;
+                            } else if (!file.reliableHeadRequests) {
+                                // Send a dummy HEAD request with range protocol
+                                //          -> good IFF status is 206 and contentLenght is present
+                                const head = new XMLHttpRequest();
+                                if (file.dataProtocol == DuckDBDataProtocol.S3) {
+                                    head.open('HEAD', getHTTPUrl(file.s3Config, file.dataUrl!), false);
+                                    addS3Headers(head, file.s3Config, file.dataUrl!, 'HEAD');
+                                } else {
+                                    head.open('HEAD', file.dataUrl!, false);
+                                }
+                                head.setRequestHeader('Range', `bytes=0-`);
+                                head.send(null);
+
+                                // Supports range requests
+                                contentLength = head.getResponseHeader('Content-Length');
+                                if (contentLength !== null && +contentLength > 1) {
+                                    presumedLength = contentLength;
+                                }
                             }
 
                             if (xhr.status == 206 && contentLength2 !== null && +contentLength2 == 1 && presumedLength !== null) {

packages/duckdb-wasm/src/bindings/web_file.ts

@@ -8,5 +8,6 @@ export interface WebFile {
     dataUrl?: string;
     dataNativeFd?: number;
     collectStatistics?: boolean;
+    reliableHeadRequests?: boolean;
     allowFullHttpReads?: boolean;
 }