From a1865ea9c4673f153d76300d8276c31d9781c88d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 5 Jun 2024 07:23:50 +0200 Subject: [PATCH] Add setting reliableHeadRequests --- lib/include/duckdb/web/config.h | 3 ++- lib/src/config.cc | 4 ++++ lib/src/io/web_filesystem.cc | 7 +++++++ packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs | 2 ++ packages/duckdb-wasm/src/bindings/config.ts | 1 + packages/duckdb-wasm/src/bindings/runtime.ts | 2 ++ packages/duckdb-wasm/src/bindings/runtime_browser.ts | 4 ++-- packages/duckdb-wasm/src/bindings/web_file.ts | 1 + 8 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/include/duckdb/web/config.h b/lib/include/duckdb/web/config.h index 527dbd31b..d546c02fa 100644 --- a/lib/include/duckdb/web/config.h +++ b/lib/include/duckdb/web/config.h @@ -70,6 +70,7 @@ struct DuckDBConfigOptions { struct FileSystemConfig { /// Allow falling back to full HTTP reads if the server does not support range requests std::optional allow_full_http_reads = std::nullopt; + std::optional reliableHeadRequests = std::nullopt; }; struct WebDBConfig { @@ -87,7 +88,7 @@ struct WebDBConfig { .cast_decimal_to_double = std::nullopt, }; /// The filesystem - FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt}; + FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt, .reliableHeadRequests = std::nullopt}; /// These options are fetched from DuckDB DuckDBConfigOptions duckdb_config_options = { diff --git a/lib/src/config.cc b/lib/src/config.cc index d6016e617..1c9567a34 100644 --- a/lib/src/config.cc +++ b/lib/src/config.cc @@ -49,6 +49,7 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) { .filesystem = FileSystemConfig{ .allow_full_http_reads = std::nullopt, + .reliableHeadRequests = std::nullopt, }, .duckdb_config_options = DuckDBConfigOptions{ @@ -97,6 +98,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) { if (fs.HasMember("allowFullHTTPReads") && fs["allowFullHTTPReads"].IsBool()) { config.filesystem.allow_full_http_reads = fs["allowFullHTTPReads"].GetBool(); } + if (fs.HasMember("reliableHeadRequests") && fs["reliableHeadRequests"].IsBool()) { + config.filesystem.reliableHeadRequests = fs["reliableHeadRequests"].GetBool(); + } } } if (!config.query.cast_bigint_to_double.has_value()) { diff --git a/lib/src/io/web_filesystem.cc b/lib/src/io/web_filesystem.cc index cedb3e717..b9d1efcf4 100644 --- a/lib/src/io/web_filesystem.cc +++ b/lib/src/io/web_filesystem.cc @@ -319,6 +319,10 @@ rapidjson::Value WebFileSystem::WebFile::WriteInfo(rapidjson::Document &doc) con filesystem_.config_->filesystem.allow_full_http_reads.value_or(true)) { value.AddMember("allowFullHttpReads", true, allocator); } + if ((data_protocol_ == DataProtocol::HTTP || data_protocol_ == DataProtocol::S3) && + filesystem_.config_->filesystem.reliableHeadRequests.value_or(true)) { + value.AddMember("reliableHeadRequests", true, allocator); + } value.AddMember("collectStatistics", filesystem_.file_statistics_->TracksFile(file_name_), doc.GetAllocator()); if (data_protocol_ == DataProtocol::S3) { @@ -493,6 +497,9 @@ rapidjson::Value WebFileSystem::WriteGlobalFileInfo(rapidjson::Document &doc, ui if (config_->filesystem.allow_full_http_reads.value_or(true)) { value.AddMember("allowFullHttpReads", true, allocator); } + if (config_->filesystem.reliableHeadRequests.value_or(true)) { + value.AddMember("reliableHeadRequests", true, allocator); + } value.AddMember("s3Config", writeS3Config(config_->duckdb_config_options, allocator), allocator); diff --git a/packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs b/packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs index f51bb2c68..7b2ea99aa 100644 --- a/packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs +++ b/packages/duckdb-wasm-shell/crate/src/duckdb/web_file.rs @@ -22,6 +22,8 @@ pub struct WebFile { pub data_url: Option, #[serde(rename = "dataNativeFd")] pub data_native_fd: Option, + #[serde(rename = "reliableHeadRequests")] + pub allow_full_http_reads: Option, #[serde(rename = "allowFullHttpReads")] pub allow_full_http_reads: Option, #[serde(rename = "collectStatistics")] diff --git a/packages/duckdb-wasm/src/bindings/config.ts b/packages/duckdb-wasm/src/bindings/config.ts index ba9033e4c..b2de071af 100644 --- a/packages/duckdb-wasm/src/bindings/config.ts +++ b/packages/duckdb-wasm/src/bindings/config.ts @@ -25,6 +25,7 @@ export interface DuckDBFilesystemConfig { /** * Allow falling back to full HTTP reads if the server does not support range requests. */ + reliableHeadRequests?: boolean; allowFullHTTPReads?: boolean; } diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index 2b137c7a0..83632deca 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -76,6 +76,7 @@ export interface DuckDBFileInfo { fileName: string; dataProtocol: DuckDBDataProtocol; dataUrl: string | null; + reliableHeadRequests?: boolean; allowFullHttpReads?: boolean; s3Config?: S3Config; } @@ -83,6 +84,7 @@ export interface DuckDBFileInfo { /** Global info for all files registered with DuckDB */ export interface DuckDBGlobalFileInfo { cacheEpoch: number; + reliableHeadRequests?: boolean; allowFullHttpReads?: boolean; s3Config?: S3Config; } diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index b3375e57a..2c0c270d3 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -159,7 +159,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { // Supports ranges? let contentLength = null; let error: any | null = null; - if (!file.allowFullHttpReads) { + if (file.reliableHeadRequests || !file.allowFullHttpReads) { try { // Send a dummy HEAD request with range protocol // -> good IFF status is 206 and contentLenght is present @@ -211,7 +211,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { let presumedLength = null; if (contentRange !== undefined) { presumedLength = contentRange; - } else { + } else if (!file.reliableHeadRequests) { // Send a dummy HEAD request with range protocol // -> good IFF status is 206 and contentLenght is present const head = new XMLHttpRequest(); diff --git a/packages/duckdb-wasm/src/bindings/web_file.ts b/packages/duckdb-wasm/src/bindings/web_file.ts index 5306c3684..4bc909e48 100644 --- a/packages/duckdb-wasm/src/bindings/web_file.ts +++ b/packages/duckdb-wasm/src/bindings/web_file.ts @@ -8,5 +8,6 @@ export interface WebFile { dataUrl?: string; dataNativeFd?: number; collectStatistics?: boolean; + reliableHeadRequests?: boolean; allowFullHttpReads?: boolean; }