handle params better

rust/src/dataset.rs

@@ -434,12 +434,10 @@ impl Dataset {
         };
 
         // Need to include params here because it might include a commit mechanism.
-        let (object_store, _) = ObjectStore::from_uri_and_params(
-            &self.object_store.base_uri(),
-            params.store_params.clone().unwrap_or_default(),
-        )
-        .await?;
-        let object_store = Arc::new(object_store);
+        let object_store = Arc::new(
+            self.object_store()
+                .with_params(&params.store_params.clone().unwrap_or_default()),
+        );
 
         let (stream, schema) = reader_to_stream(batches)?;
 

rust/src/io/commit.rs

@@ -807,6 +807,8 @@ mod tests {
                 }
                 _ => unreachable!(),
             }
+
+            dataset.validate().await.unwrap()
         }
     }
 }

rust/src/io/object_store.rs

@@ -258,6 +258,21 @@ impl ObjectStore {
         ))
     }
 
+    pub fn with_params(&self, params: &ObjectStoreParams) -> Self {
+        Self {
+            inner: params
+                .object_store_wrapper
+                .as_ref()
+                .map(|w| w.wrap(self.inner.clone()))
+                .unwrap_or_else(|| self.inner.clone()),
+            commit_handler: params
+                .commit_handler
+                .clone()
+                .unwrap_or(self.commit_handler.clone()),
+            ..self.clone()
+        }
+    }
+
     fn new_from_path(str_path: &str, params: &ObjectStoreParams) -> Result<(Self, Path)> {
         let expanded = tilde(str_path).to_string();
         let expanded_path = StdPath::new(&expanded);
@@ -371,10 +386,6 @@ impl ObjectStore {
         &self.base_path
     }
 
-    pub fn base_uri(&self) -> String {
-        format!("{}://{}", self.scheme, self.base_path)
-    }
-
     /// Open a file for path.
     ///
     /// Parameters