feat(bindings/c): add writer operation for Bindings C and Go (#5141)

* feat(bindings/c): add writer operation

Signed-off-by: Hanchin Hsieh <[email protected]>

* feat(bindings/go): add writer operation

Signed-off-by: Hanchin Hsieh <[email protected]>

* ci(bindings/go): test against service-fs

Signed-off-by: Hanchin Hsieh <[email protected]>

* fix(bindings/go): fix tests

Signed-off-by: Hanchin Hsieh <[email protected]>

* fix(bindings/c): use BlockWriter instead of StdWriter

Signed-off-by: Hanchin Hsieh <[email protected]>

---------

Signed-off-by: Hanchin Hsieh <[email protected]>

.github/scripts/test_go_binding/matrix.yaml

@@ -22,5 +22,5 @@ build:
     goos: "linux"
     goarch: "amd64"
 service:
-  - "memory"
+  - "fs"
 

.github/workflows/ci_bindings_go.yml

@@ -127,5 +127,6 @@ jobs:
       - name: Run tests
         env:
           OPENDAL_TEST: ${{ matrix.service }}
+          OPENDAL_FS_ROOT: "/tmp/opendal/"
         working-directory: bindings/go/tests/behavior_tests
         run: CGO_ENABLE=0 go test -v -run TestBehavior

bindings/c/include/opendal.h

@@ -146,6 +146,12 @@ typedef struct BlockingLister BlockingLister;
  */
 typedef struct BlockingOperator BlockingOperator;
 
+/**
+ * BlockingWriter is designed to write data into given path in an blocking
+ * manner.
+ */
+typedef struct BlockingWriter BlockingWriter;
+
 /**
  * Entry returned by [`Lister`] or [`BlockingLister`] to represent a path and it's relative metadata.
  *
@@ -430,6 +436,31 @@ typedef struct opendal_result_operator_reader {
   struct opendal_error *error;
 } opendal_result_operator_reader;
 
+/**
+ * \brief The result type returned by opendal's writer operation.
+ * \note The opendal_writer actually owns a pointer to
+ * a opendal::BlockingWriter, which is inside the Rust core code.
+ */
+typedef struct opendal_writer {
+  struct BlockingWriter *inner;
+} opendal_writer;
+
+/**
+ * \brief The result type returned by opendal_operator_writer().
+ * The result type for opendal_operator_writer(), the field `writer` contains the writer
+ * of the path, which is an iterator of the objects under the path. the field `code` represents
+ */
+typedef struct opendal_result_operator_writer {
+  /**
+   * The pointer for opendal_writer
+   */
+  struct opendal_writer *writer;
+  /**
+   * The error, if ok, it is null
+   */
+  struct opendal_error *error;
+} opendal_result_operator_writer;
+
 /**
  * \brief The result type returned by opendal_operator_is_exist().
  *
@@ -680,6 +711,22 @@ typedef struct opendal_result_reader_read {
   struct opendal_error *error;
 } opendal_result_reader_read;
 
+/**
+ * \brief The result type returned by opendal_writer_write().
+ * The result type contains a size field, which is the size of the data written,
+ * which is zero on error. The error field is the error code and error message.
+ */
+typedef struct opendal_result_writer_write {
+  /**
+   * The write size if succeed.
+   */
+  uintptr_t size;
+  /**
+   * The error, if ok, it is null
+   */
+  struct opendal_error *error;
+} opendal_result_writer_write;
+
 #ifdef __cplusplus
 extern "C" {
 #endif // __cplusplus
@@ -970,6 +1017,47 @@ struct opendal_result_read opendal_operator_read(const struct opendal_operator *
 struct opendal_result_operator_reader opendal_operator_reader(const struct opendal_operator *op,
                                                               const char *path);
 
+/**
+ * \brief Blockingly create a writer for the specified path.
+ *
+ * This function prepares a writer that can be used to write data to the specified path
+ * using the provided operator. If successful, it returns a valid writer; otherwise, it
+ * returns an error.
+ *
+ * @param op The opendal_operator created previously
+ * @param path The designated path where the writer will be used
+ * @see opendal_operator
+ * @see opendal_result_operator_writer
+ * @see opendal_error
+ * @return Returns opendal_result_operator_writer, containing a writer and an opendal_error.
+ * If the operation succeeds, the `writer` field holds a valid writer and the `error` field
+ * is null. Otherwise, the `writer` will be null and the `error` will be set correspondingly.
+ *
+ * # Example
+ *
+ * Following is an example
+ * ```C
+ * //...prepare your opendal_operator, named op for example
+ *
+ * opendal_result_operator_writer result = opendal_operator_writer(op, "/testpath");
+ * assert(result.error == NULL);
+ * opendal_writer *writer = result.writer;
+ * // Use the writer to write data...
+ * ```
+ *
+ * # Safety
+ *
+ * It is **safe** under the cases below
+ * * The memory pointed to by `path` must contain a valid nul terminator at the end of
+ *   the string.
+ *
+ * # Panic
+ *
+ * * If the `path` points to NULL, this function panics, i.e. exits with information
+ */
+struct opendal_result_operator_writer opendal_operator_writer(const struct opendal_operator *op,
+                                                              const char *path);
+
 /**
  * \brief Blockingly delete the object in `path`.
  *
@@ -1419,6 +1507,18 @@ struct opendal_result_reader_read opendal_reader_read(const struct opendal_reade
  */
 void opendal_reader_free(struct opendal_reader *ptr);
 
+/**
+ * \brief Write data to the writer.
+ */
+struct opendal_result_writer_write opendal_writer_write(const struct opendal_writer *writer,
+                                                        struct opendal_bytes bytes);
+
+/**
+ * \brief Frees the heap memory used by the opendal_writer.
+ * \note This function make sure all data have been stored.
+ */
+void opendal_writer_free(struct opendal_writer *ptr);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus

bindings/c/src/lib.rs

@@ -51,9 +51,11 @@ pub use result::opendal_result_list;
 pub use result::opendal_result_lister_next;
 pub use result::opendal_result_operator_new;
 pub use result::opendal_result_operator_reader;
+pub use result::opendal_result_operator_writer;
 pub use result::opendal_result_read;
 pub use result::opendal_result_reader_read;
 pub use result::opendal_result_stat;
+pub use result::opendal_result_writer_write;
 
 mod types;
 pub use types::opendal_bytes;
@@ -64,3 +66,6 @@ pub use entry::opendal_entry;
 
 mod reader;
 pub use reader::opendal_reader;
+
+mod writer;
+pub use writer::opendal_writer;

bindings/c/src/operator.rs

@@ -391,6 +391,69 @@ pub unsafe extern "C" fn opendal_operator_reader(
     }
 }
 
+/// \brief Blockingly create a writer for the specified path.
+///
+/// This function prepares a writer that can be used to write data to the specified path
+/// using the provided operator. If successful, it returns a valid writer; otherwise, it
+/// returns an error.
+///
+/// @param op The opendal_operator created previously
+/// @param path The designated path where the writer will be used
+/// @see opendal_operator
+/// @see opendal_result_operator_writer
+/// @see opendal_error
+/// @return Returns opendal_result_operator_writer, containing a writer and an opendal_error.
+/// If the operation succeeds, the `writer` field holds a valid writer and the `error` field
+/// is null. Otherwise, the `writer` will be null and the `error` will be set correspondingly.
+///
+/// # Example
+///
+/// Following is an example
+/// ```C
+/// //...prepare your opendal_operator, named op for example
+///
+/// opendal_result_operator_writer result = opendal_operator_writer(op, "/testpath");
+/// assert(result.error == NULL);
+/// opendal_writer *writer = result.writer;
+/// // Use the writer to write data...
+/// ```
+///
+/// # Safety
+///
+/// It is **safe** under the cases below
+/// * The memory pointed to by `path` must contain a valid nul terminator at the end of
+///   the string.
+///
+/// # Panic
+///
+/// * If the `path` points to NULL, this function panics, i.e. exits with information
+#[no_mangle]
+pub unsafe extern "C" fn opendal_operator_writer(
+    op: *const opendal_operator,
+    path: *const c_char,
+) -> opendal_result_operator_writer {
+    if path.is_null() {
+        panic!("The path given is pointing at NULL");
+    }
+    let op = (*op).as_ref();
+
+    let path = unsafe { std::ffi::CStr::from_ptr(path).to_str().unwrap() };
+    let writer = match op.writer(path) {
+        Ok(writer) => writer,
+        Err(err) => {
+            return opendal_result_operator_writer {
+                writer: std::ptr::null_mut(),
+                error: opendal_error::new(err),
+            }
+        }
+    };
+
+    opendal_result_operator_writer {
+        writer: Box::into_raw(Box::new(opendal_writer::new(writer))),
+        error: std::ptr::null_mut(),
+    }
+}
+
 /// \brief Blockingly delete the object in `path`.
 ///
 /// Delete the object in `path` blockingly by `op_ptr`.

bindings/c/src/result.rs

@@ -131,3 +131,25 @@ pub struct opendal_result_reader_read {
     /// The error, if ok, it is null
     pub error: *mut opendal_error,
 }
+
+/// \brief The result type returned by opendal_operator_writer().
+/// The result type for opendal_operator_writer(), the field `writer` contains the writer
+/// of the path, which is an iterator of the objects under the path. the field `code` represents
+#[repr(C)]
+pub struct opendal_result_operator_writer {
+    /// The pointer for opendal_writer
+    pub writer: *mut opendal_writer,
+    /// The error, if ok, it is null
+    pub error: *mut opendal_error,
+}
+
+/// \brief The result type returned by opendal_writer_write().
+/// The result type contains a size field, which is the size of the data written,
+/// which is zero on error. The error field is the error code and error message.
+#[repr(C)]
+pub struct opendal_result_writer_write {
+    /// The write size if succeed.
+    pub size: usize,
+    /// The error, if ok, it is null
+    pub error: *mut opendal_error,
+}

bindings/c/src/writer.rs

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use ::opendal as core;
+
+use super::*;
+
+/// \brief The result type returned by opendal's writer operation.
+/// \note The opendal_writer actually owns a pointer to
+/// a opendal::BlockingWriter, which is inside the Rust core code.
+#[repr(C)]
+pub struct opendal_writer {
+    inner: *mut core::BlockingWriter,
+}
+
+impl opendal_writer {
+    pub(crate) fn new(writer: core::BlockingWriter) -> Self {
+        Self {
+            inner: Box::into_raw(Box::new(writer)),
+        }
+    }
+
+    /// \brief Write data to the writer.
+    #[no_mangle]
+    pub unsafe extern "C" fn opendal_writer_write(
+        writer: *const Self,
+        bytes: opendal_bytes,
+    ) -> opendal_result_writer_write {
+        let inner = unsafe { &mut *(*writer).inner };
+        let size = bytes.len;
+        match inner.write(bytes) {
+            Ok(()) => opendal_result_writer_write {
+                size,
+                error: std::ptr::null_mut(),
+            },
+            Err(e) => opendal_result_writer_write {
+                size: 0,
+                error: opendal_error::new(
+                    core::Error::new(core::ErrorKind::Unexpected, "write failed from writer")
+                        .set_source(e),
+                ),
+            },
+        }
+    }
+
+    /// \brief Frees the heap memory used by the opendal_writer.
+    /// \note This function make sure all data have been stored.
+    #[no_mangle]
+    pub unsafe extern "C" fn opendal_writer_free(ptr: *mut opendal_writer) {
+        if !ptr.is_null() {
+            let mut w = unsafe { Box::from_raw((*ptr).inner) };
+            let _ = w.close();
+            let _ = unsafe { Box::from_raw(ptr) };
+        }
+    }
+}

bindings/c/tests/bdd.cpp

@@ -90,6 +90,20 @@ TEST_F(OpendalBddTest, FeatureTest)
         EXPECT_EQ(this->content[i], (char)(r.data->data[i]));
     }
 
+    // The blocking file should be deleted
+    error = opendal_operator_delete(this->p, this->path.c_str());
+    EXPECT_EQ(error, nullptr);
+    e = opendal_operator_is_exist(this->p, this->path.c_str());
+    EXPECT_EQ(e.error, nullptr);
+    EXPECT_FALSE(e.is_exist);
+
+    opendal_result_operator_writer writer = opendal_operator_writer(this->p, this->path.c_str());
+    EXPECT_EQ(writer.error, nullptr);
+    opendal_result_writer_write w = opendal_writer_write(writer.writer, data);
+    EXPECT_EQ(w.error, nullptr);
+    EXPECT_EQ(w.size, this->content.length());
+    opendal_writer_free(writer.writer);
+
     // The blocking file "test" must have content "Hello, World!" and read into buffer
     int length = this->content.length();
     unsigned char buffer[this->content.length()];
@@ -102,13 +116,6 @@ TEST_F(OpendalBddTest, FeatureTest)
     }
     opendal_reader_free(reader.reader);
 
-    // The blocking file should be deleted
-    error = opendal_operator_delete(this->p, this->path.c_str());
-    EXPECT_EQ(error, nullptr);
-    e = opendal_operator_is_exist(this->p, this->path.c_str());
-    EXPECT_EQ(e.error, nullptr);
-    EXPECT_FALSE(e.is_exist);
-
     // The deletion operation should be idempotent
     error = opendal_operator_delete(this->p, this->path.c_str());
     EXPECT_EQ(error, nullptr);

bindings/go/ffi.go

@@ -140,4 +140,8 @@ var withFFIs = []contextWithFFI{
 	withOperatorReader,
 	withReaderRead,
 	withReaderFree,
+
+	withOperatorWriter,
+	withWriterWrite,
+	withWriterFree,
 }

bindings/go/tests/behavior_tests/go.mod

@@ -20,7 +20,8 @@ module opendal_test
 go 1.22.5
 
 require (
-	github.com/apache/opendal-go-services/memory v0.0.0-20240719030108-74ff217cfef9
+	github.com/apache/opendal-go-services/fs v0.1.3
+	github.com/apache/opendal-go-services/memory v0.1.3
 	github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24
 	github.com/google/uuid v1.6.0
 	github.com/stretchr/testify v1.9.0
@@ -30,7 +31,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ebitengine/purego v0.7.1 // indirect
 	github.com/jupiterrider/ffi v0.1.0-beta.9 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.10 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect