feat(integrations/cloudfilter): introduce behavior tests

.github/scripts/test_behavior/plan.py

@@ -35,6 +35,7 @@
 
 BIN = ["ofs"]
 
+INTEGRATION = ["cloudfilter"]
 
 def provided_cases() -> list[dict[str, str]]:
     root_dir = f"{GITHUB_DIR}/services"
@@ -86,6 +87,8 @@ class Hint:
     binding_nodejs: bool = field(default=False, init=False)
     # Is bin ofs affected?
     bin_ofs: bool = field(default=False, init=False)
+    # Is integration cloudfilter affected?
+    integration_cloudfilter: bool = field(default=False, init=False)
 
     # Should we run all services tests?
     all_service: bool = field(default=False, init=False)
@@ -133,6 +136,7 @@ def calculate_hint(changed_files: list[str]) -> Hint:
             hint.binding_python = True
             hint.binding_nodejs = True
             hint.bin_ofs = True
+            hint.integration_cloudfilter = True
             hint.all_service = True
 
         # language binding affected
@@ -147,6 +151,12 @@ def calculate_hint(changed_files: list[str]) -> Hint:
                 setattr(hint, f"bin_{bin}", True)
                 hint.all_service = True
 
+        # integration affected
+        for integration in INTEGRATION:
+            if p.startswith(f"integrations/{integration}"):
+                setattr(hint, f"integration_{integration}", True)
+                hint.all_service = True
+
         # core service affected
         match = re.search(r"core/src/services/([^/]+)/", p)
         if match:
@@ -155,6 +165,8 @@ def calculate_hint(changed_files: list[str]) -> Hint:
                 setattr(hint, f"binding_{language}", True)
             for bin in BIN:
                 setattr(hint, f"bin_{bin}", True)
+            for integration in INTEGRATION:
+                setattr(hint, f"integration_{integration}", True)
             hint.services.add(match.group(1))
 
         # core test affected
@@ -165,6 +177,8 @@ def calculate_hint(changed_files: list[str]) -> Hint:
                 setattr(hint, f"binding_{language}", True)
             for bin in BIN:
                 setattr(hint, f"bin_{bin}", True)
+            for integration in INTEGRATION:
+                setattr(hint, f"integration_{integration}", True)
             hint.services.add(match.group(1))
 
         # fixture affected
@@ -175,6 +189,8 @@ def calculate_hint(changed_files: list[str]) -> Hint:
                 setattr(hint, f"binding_{language}", True)
             for bin in BIN:
                 setattr(hint, f"bin_{bin}", True)
+            for integration in INTEGRATION:
+                setattr(hint, f"integration_{integration}", True)
             hint.services.add(match.group(1))
 
     return hint
@@ -315,6 +331,23 @@ def plan(changed_files: list[str]) -> dict[str, Any]:
         if len(bin_cases) > 0:
             jobs["components"][f"bin_{bin}"] = True
             jobs[f"bin_{bin}"].append({"os": "ubuntu-latest", "cases": bin_cases})
+
+    for integration in INTEGRATION:
+        jobs[f"integration_{integration}"] = []
+        jobs["components"][f"integration_{integration}"] = False
+
+        # cloudfilter is the only integration need to run upon windows, let's hard code it here.
+        if integration == "cloudfilter" and (getattr(hint, "integration_cloudfilter") or hint.all_service):
+            jobs["components"][f"integration_cloudfilter"] = True
+            jobs[f"integration_cloudfilter"] = [
+                {
+                    "os": "windows-latest", "cases": [{
+                        "setup": "fixture_data",
+                        "service": "fs",
+                        "feature": "services-fs"
+                    }]
+                },
+            ]
     return jobs
 
 

.github/scripts/test_behavior/test_plan.py

@@ -59,6 +59,26 @@ def test_bin_ofs(self):
         cases = [v["service"] for v in result["bin_ofs"][0]["cases"]]
         # Should contain ofs
         self.assertTrue("fs" in cases)
+
+    def test_integration_cloudfilter(self):
+        result = plan(["integrations/cloudfilter/Cargo.toml"])
+        self.assertTrue(result["components"]["integration_cloudfilter"])
+        self.assertEqual(result["integration_cloudfilter"][0]["cases"], 
+                        [{"service": "fs", "setup": "fixture_data", "feature": "services-fs"}])
+
+        result = plan(["core/src/services/fs/mod.rs"])
+        self.assertTrue(result["components"]["integration_cloudfilter"])
+        self.assertEqual(result["integration_cloudfilter"][0]["cases"], 
+                        [{"service": "fs", "setup": "fixture_data", "feature": "services-fs"}])
+
+        result = plan(["core/src/services/s3/mod.rs"])
+        self.assertTrue(result["components"]["integration_cloudfilter"])
+        self.assertEqual(result["integration_cloudfilter"][0]["cases"], 
+                        [{"service": "fs", "setup": "fixture_data", "feature": "services-fs"}])
+
+        result = plan(["bindings/java/pom.xml"])
+        self.assertFalse(result["components"]["integration_cloudfilter"])
+        self.assertTrue(len(result["integration_cloudfilter"]) == 0)
 
 
 if __name__ == "__main__":

.github/workflows/test_behavior.yml

@@ -145,3 +145,17 @@ jobs:
     with:
       os: ${{ matrix.os }}
       cases: ${{ toJson(matrix.cases) }}
+
+  test_integration_cloudfilter:
+    name: integration_cloudfilter / ${{ matrix.os }}
+    needs: [ plan ]
+    if: fromJson(needs.plan.outputs.plan).components.integration_cloudfilter
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.plan.outputs.plan).integration_cloudfilter }}
+    uses: ./.github/workflows/test_behavior_integration_cloudfilter.yml
+    with:
+      os: ${{ matrix.os }}
+      cases: ${{ toJson(matrix.cases) }}

.github/workflows/test_behavior_integration_cloudfilter.yml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Behavior Test Integration CloudFilter
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      cases:
+        required: true
+        type: string
+
+jobs:
+  test:
+    name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+    runs-on: ${{ inputs.os }}
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        cases: ${{ fromJson(inputs.cases) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-protoc: true
+          need-rocksdb: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # TODO: 1password is only supported on linux
+      #
+      # Waiting for https://github.com/1Password/load-secrets-action/issues/46
+      - name: Setup 1Password Connect
+        if: runner.os == 'Linux'
+        uses: 1password/load-secrets-action/configure@v1
+        with:
+          connect-host: ${{ secrets.OP_CONNECT_HOST }}
+          connect-token: ${{ secrets.OP_CONNECT_TOKEN }}
+
+      - name: Test Integration CloudFilter
+        working-directory: integrations/cloudfilter
+        run: cargo test --test behavior
+        env:
+          OPENDAL_TEST: fs
+          OPENDAL_FS_ROOT: ../../fixtures/data
+          OPENDAL_DISABLE_RANDOM_ROOT: 'true'

integrations/cloudfilter/Cargo.toml

@@ -29,7 +29,7 @@ version = "0.0.0"
 [dependencies]
 anyhow = "1.0.86"
 bincode = "1.3.3"
-cloud-filter = "0.0.3"
+cloud-filter = "0.0.5"
 futures = "0.3.30"
 log = "0.4.17"
 opendal = { version = "0.48.0", path = "../../core" }
@@ -39,9 +39,17 @@ serde = { version = "1.0.203", features = ["derive"] }
 env_logger = "0.11.2"
 opendal = { version = "0.48.0", path = "../../core", features = [
   "services-fs",
+  "tests",
 ] }
 tokio = { version = "1.38.0", features = [
   "macros",
   "rt-multi-thread",
   "signal",
 ] }
+libtest-mimic = "0.7.3"
+powershell_script = "1.1.0"
+
+[[test]]
+harness = false
+name = "behavior"
+path = "tests/behavior/main.rs"

integrations/cloudfilter/src/lib.rs

@@ -25,11 +25,10 @@ use std::{
 
 use cloud_filter::{
     error::{CResult, CloudErrorKind},
-    filter::{info, ticket, Filter},
+    filter::{info, ticket, Filter, Request},
     metadata::Metadata,
     placeholder::{ConvertOptions, Placeholder},
     placeholder_file::PlaceholderFile,
-    request::Request,
     utility::{FileTime, WriteAt},
 };
 use file::FileBlob;

integrations/cloudfilter/tests/behavior/README.md

@@ -0,0 +1,12 @@
+# Behavior tests for OpenDAL™ Cloud Filter Integration
+
+Behavior tests are used to make sure every service works correctly.
+
+`cloudfilter_opendal` is readonly currently, so we assume `fixtures/data` is the root of the test data.
+
+## Run
+
+```pwsh
+cd .\integrations\cloudfilter
+$env:OPENDAL_TEST='fs'; $env:OPENDAL_FS_ROOT='../../fixtures/data'; $env:OPENDAL_DISABLE_RANDOM_ROOT='true'; cargo test --test behavior
+```

integrations/cloudfilter/tests/behavior/fetch_data.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use libtest_mimic::Failed;
+
+use crate::{
+    utils::{file_content, file_length},
+    ROOT_PATH,
+};
+
+pub fn test_fetch_data() -> Result<(), Failed> {
+    let files = [
+        (
+            "normal_file.txt",
+            include_str!("..\\..\\..\\..\\fixtures/data/normal_file.txt"),
+        ),
+        (
+            "special_file  !@#$%^&()_+-=;',.txt",
+            include_str!("..\\..\\..\\..\\fixtures/data/special_file  !@#$%^&()_+-=;',.txt"),
+        ),
+    ];
+    for (file, expected_content) in files {
+        let path = format!("{ROOT_PATH}\\{file}");
+        assert_eq!(
+            expected_content.len(),
+            file_length(&path).expect("file length"),
+            "file length",
+        );
+
+        assert_eq!(
+            expected_content,
+            file_content(&path).expect("file hash"),
+            "file hash",
+        )
+    }
+    Ok(())
+}

integrations/cloudfilter/tests/behavior/fetch_placeholder.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use libtest_mimic::Failed;
+
+use crate::{utils::list, ROOT_PATH};
+
+pub fn test_fetch_placeholder() -> Result<(), Failed> {
+    let files = ["normal_file.txt", "special_file  !@#$%^&()_+-=;',.txt"];
+    let dirs = ["normal_dir", "special_dir  !@#$%^&()_+-=;',"];
+
+    assert_eq!(
+        list(ROOT_PATH, "File").expect("list files"),
+        files,
+        "list files"
+    );
+    assert_eq!(
+        list(ROOT_PATH, "Directory").expect("list dirs"),
+        dirs,
+        "list dirs"
+    );
+
+    Ok(())
+}