Skip to content

Commit

Permalink
Port more tests to REST CI (#4838)
Browse files Browse the repository at this point in the history 
As per title, more tests in tiledb_unit are adapted to run on the REST
CI runner.

After this PR is merged we'll have 160 tests ported to REST-CI runner,
of which 143 will be running and passing and 17 will be disabled until
the issues found and logged
here:https://app.shortcut.com/tiledb-inc/story/40489/issues-found-while-running-tiledb-unit-core-tests-against-rest-cloud-server
are fixed.

---
TYPE: NO_HISTORY
DESC: Port more tests to REST CI
  • Loading branch information
@ypatia
ypatia authored Apr 9, 2024
1 parent adafa1e commit e0bc0dd
Show file tree
Hide file tree
Showing 27 changed files with 1,850 additions and 4,525 deletions.
81 changes: 38 additions & 43 deletions test/src/cpp-integration-query-condition.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include <iostream>
#include <vector>

#include <test/support/src/vfs_helpers.h>
#include <test/support/tdb_catch.h>
#include "tiledb/sm/cpp_api/tiledb"
#include "tiledb/sm/misc/utils.h"
Expand Down Expand Up @@ -91,14 +92,16 @@ inline int index_from_row_col(int r, int c) {
* @param set_dups Whether the array allows coordinate duplicates.
* @param a_data_read Data buffer to store cell values on attribute a.
* @param b_data_read Data buffer to store cell values on attribute b.
* @param array_uri URI of array to create.
*/
void create_array(
Context& ctx,
tiledb_array_type_t array_type,
bool set_dups,
bool add_utf8_attr,
std::vector<int>& a_data_read,
std::vector<float>& b_data_read) {
std::vector<float>& b_data_read,
const std::string& array_uri = array_name) {
Domain domain(ctx);
domain.add_dimension(Dimension::create<int>(ctx, "rows", {{1, num_rows}}, 4))
.add_dimension(Dimension::create<int>(ctx, "cols", {{1, num_rows}}, 4));
Expand All @@ -123,7 +126,7 @@ void create_array(
attr_c.set_nullable(true);
schema.add_attribute(attr_c);
}
Array::create(array_name, schema);
Array::create(array_uri, schema);

// Write some initial data and close the array.
std::vector<int> row_dims;
Expand Down Expand Up @@ -179,7 +182,7 @@ void create_array(
}

if (array_type == TILEDB_SPARSE) {
Array array_w(ctx, array_name, TILEDB_WRITE);
Array array_w(ctx, array_uri, TILEDB_WRITE);
Query query_w(ctx, array_w);
query_w.set_layout(TILEDB_UNORDERED)
.set_data_buffer("rows", row_dims)
Expand All @@ -197,7 +200,7 @@ void create_array(
query_w.finalize();
array_w.close();
} else if (array_type == TILEDB_DENSE) {
Array array_w(ctx, array_name, TILEDB_WRITE);
Array array_w(ctx, array_uri, TILEDB_WRITE);
Query query_w(ctx, array_w);
query_w.set_layout(TILEDB_ROW_MAJOR)
.set_data_buffer("a", a_data)
Expand All @@ -215,7 +218,7 @@ void create_array(
}

// Open and read the entire array to save data for future comparisons.
Array array1(ctx, array_name, TILEDB_READ);
Array array1(ctx, array_uri, TILEDB_READ);
Query query1(ctx, array1);
query1.set_layout(TILEDB_ROW_MAJOR)
.set_data_buffer("a", a_data_read)
Expand Down Expand Up @@ -316,15 +319,12 @@ struct TestParams {

TEST_CASE(
"Testing read query with empty QC, with no range.",
"[query][query-condition][empty]") {
"[query][query-condition][empty][rest]") {
// Initial setup.
std::srand(static_cast<uint32_t>(time(0)));
Context ctx;
VFS vfs(ctx);

if (vfs.is_dir(array_name)) {
vfs.remove_dir(array_name);
}
test::VFSTestSetup vfs_test_setup;
Context ctx{vfs_test_setup.ctx()};
auto array_uri{vfs_test_setup.array_uri(array_name)};

// Create an empty query condition
QueryCondition qc(ctx);
Expand All @@ -351,16 +351,19 @@ TEST_CASE(
params.set_dups_,
params.add_utf8_attr_,
a_data_read,
b_data_read);
b_data_read,
array_uri);

// Create the query, which reads over the entire array with query condition
Config config;
if (params.legacy_) {
config.set("sm.query.sparse_global_order.reader", "legacy");
config.set("sm.query.sparse_unordered_with_dups.reader", "legacy");
}
Context ctx2 = Context(config);
Array array(ctx2, array_name, TILEDB_READ);

auto vfs_test_setup2 = tiledb::test::VFSTestSetup(config.ptr().get(), false);
auto ctx2 = vfs_test_setup2.ctx();
Array array(ctx2, array_uri, TILEDB_READ);
Query query(ctx2, array);

// Set a subarray for dense.
Expand All @@ -378,15 +381,12 @@ TEST_CASE(

TEST_CASE(
"Testing read query with basic QC, with no range.",
"[query][query-condition]") {
"[query][query-condition][rest]") {
// Initial setup.
std::srand(static_cast<uint32_t>(time(0)));
Context ctx;
VFS vfs(ctx);

if (vfs.is_dir(array_name)) {
vfs.remove_dir(array_name);
}
test::VFSTestSetup vfs_test_setup;
Context ctx{vfs_test_setup.ctx()};
auto array_uri{vfs_test_setup.array_uri(array_name)};

// Define query condition (b < 4.0).
QueryCondition qc(ctx);
Expand Down Expand Up @@ -415,7 +415,8 @@ TEST_CASE(
params.set_dups_,
params.add_utf8_attr_,
a_data_read,
b_data_read);
b_data_read,
array_uri);

// Create the query, which reads over the entire array with query condition
// (b < 4.0).
Expand All @@ -424,8 +425,9 @@ TEST_CASE(
config.set("sm.query.sparse_global_order.reader", "legacy");
config.set("sm.query.sparse_unordered_with_dups.reader", "legacy");
}
Context ctx2 = Context(config);
Array array(ctx2, array_name, TILEDB_READ);
auto vfs_test_setup2 = tiledb::test::VFSTestSetup(config.ptr().get(), false);
auto ctx2 = vfs_test_setup2.ctx();
Array array(ctx2, array_uri, TILEDB_READ);
Query query(ctx2, array);

// Set a subarray for dense.
Expand Down Expand Up @@ -500,23 +502,16 @@ TEST_CASE(

query.finalize();
array.close();

if (vfs.is_dir(array_name)) {
vfs.remove_dir(array_name);
}
}

TEST_CASE(
"Testing read query with basic negated QC, with no range.",
"[query][query-condition][negation]") {
"[query][query-condition][negation][rest]") {
// Initial setup.
std::srand(static_cast<uint32_t>(time(0)));
Context ctx;
VFS vfs(ctx);

if (vfs.is_dir(array_name)) {
vfs.remove_dir(array_name);
}
test::VFSTestSetup vfs_test_setup;
Context ctx{vfs_test_setup.ctx()};
auto array_uri{vfs_test_setup.array_uri(array_name)};

// Define query condition (b < 4.0).
QueryCondition qc(ctx);
Expand Down Expand Up @@ -547,7 +542,8 @@ TEST_CASE(
params.set_dups_,
params.add_utf8_attr_,
a_data_read,
b_data_read);
b_data_read,
array_uri);

// Create the query, which reads over the entire array with query condition
// (b < 4.0).
Expand All @@ -556,8 +552,11 @@ TEST_CASE(
config.set("sm.query.sparse_global_order.reader", "legacy");
config.set("sm.query.sparse_unordered_with_dups.reader", "legacy");
}
Context ctx2 = Context(config);
Array array(ctx2, array_name, TILEDB_READ);

vfs_test_setup.update_config(config.ptr().get());
Context ctx2 = vfs_test_setup.ctx();

Array array(ctx2, array_uri, TILEDB_READ);
Query query(ctx2, array);

// Set a subarray for dense.
Expand Down Expand Up @@ -632,10 +631,6 @@ TEST_CASE(

query.finalize();
array.close();

if (vfs.is_dir(array_name)) {
vfs.remove_dir(array_name);
}
}

TEST_CASE(
Expand Down
71 changes: 22 additions & 49 deletions test/src/test-capi-consolidation-plan.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,9 @@
using namespace tiledb;
using namespace tiledb::test;

#ifndef TILEDB_TESTS_ENABLE_REST
constexpr bool rest_tests = false;
#else
constexpr bool rest_tests = true;
#endif

struct ConsolidationPlanFx {
// Constructors/destructors.
ConsolidationPlanFx();
~ConsolidationPlanFx();

// Functions.
void create_sparse_array(bool allows_dups = false, bool encrypt = false);
Expand All @@ -67,38 +60,24 @@ struct ConsolidationPlanFx {
bool is_array(const std::string& array_name);
void check_last_error(std::string expected);

// TileDB context.
Context ctx_;
// Full URI initialized using fs_vec_ random temp directory.
std::string array_name_;
VFSTestSetup vfs_test_setup_;

// Vector of supported filsystems
tiledb_vfs_handle_t* vfs_c_{nullptr};
tiledb_ctx_handle_t* ctx_c_{nullptr};
const std::vector<std::unique_ptr<test::SupportedFs>> fs_vec_;
// TileDB context
tiledb_ctx_t* ctx_c_;
std::string array_name_;
Context ctx_;

std::string key_ = "0123456789abcdeF0123456789abcdeF";
const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM;
};

ConsolidationPlanFx::ConsolidationPlanFx()
: fs_vec_(test::vfs_test_get_fs_vec()) {
ConsolidationPlanFx::ConsolidationPlanFx() {
Config config;
config.set("sm.consolidation.buffer_size", "1000");
REQUIRE(
test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_, config.ptr().get()).ok());
ctx_ = Context(ctx_c_);
std::string temp_dir = fs_vec_[0]->temp_dir();
if constexpr (rest_tests) {
array_name_ = "tiledb://unit/";
}
array_name_ += temp_dir + "test_consolidation_plan_array";
test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir);
}

ConsolidationPlanFx::~ConsolidationPlanFx() {
Array::delete_array(ctx_, array_name_);
REQUIRE(test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok());
vfs_test_setup_.update_config(config.ptr().get());
ctx_c_ = vfs_test_setup_.ctx_c;
ctx_ = vfs_test_setup_.ctx();
array_name_ = vfs_test_setup_.array_uri("test_consolidation_plan_array");
}

void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) {
Expand Down Expand Up @@ -174,7 +153,7 @@ void ConsolidationPlanFx::write_sparse(
void ConsolidationPlanFx::check_last_error(std::string expected) {
const char* msg = "unset";
tiledb_error_t* err{nullptr};
tiledb_ctx_get_last_error(ctx_.ptr().get(), &err);
tiledb_ctx_get_last_error(ctx_c_, &err);
if (err != nullptr) {
tiledb_error_message(err, &msg);
}
Expand All @@ -193,31 +172,28 @@ TEST_CASE_METHOD(

tiledb_consolidation_plan_t* consolidation_plan{};
CHECK(
TILEDB_OK == tiledb_consolidation_plan_create_with_mbr(
ctx_.ptr().get(),
array.ptr().get(),
1024 * 1024,
&consolidation_plan));
TILEDB_OK ==
tiledb_consolidation_plan_create_with_mbr(
ctx_c_, array.ptr().get(), 1024 * 1024, &consolidation_plan));

uint64_t num_nodes = 11;
CHECK(
TILEDB_OK == tiledb_consolidation_plan_get_num_nodes(
ctx_.ptr().get(), consolidation_plan, &num_nodes));
ctx_c_, consolidation_plan, &num_nodes));
CHECK(num_nodes == 0);

uint64_t num_fragments = 11;
CHECK(
TILEDB_ERR ==
tiledb_consolidation_plan_get_num_fragments(
ctx_.ptr().get(), consolidation_plan, 0, &num_fragments));
TILEDB_ERR == tiledb_consolidation_plan_get_num_fragments(
ctx_c_, consolidation_plan, 0, &num_fragments));
CHECK(num_fragments == 11);
check_last_error(
"Error: ConsolidationPlan: Trying to access a node that doesn't exist.");

const char* frag_uri = nullptr;
CHECK(
TILEDB_ERR == tiledb_consolidation_plan_get_fragment_uri(
ctx_.ptr().get(), consolidation_plan, 0, 0, &frag_uri));
ctx_c_, consolidation_plan, 0, 0, &frag_uri));
CHECK(frag_uri == nullptr);
check_last_error(
"Error: ConsolidationPlan: Trying to access a node that doesn't exist.");
Expand All @@ -236,16 +212,13 @@ TEST_CASE_METHOD(

tiledb_consolidation_plan_t* consolidation_plan{};
CHECK(
TILEDB_OK == tiledb_consolidation_plan_create_with_mbr(
ctx_.ptr().get(),
array.ptr().get(),
1024 * 1024,
&consolidation_plan));
TILEDB_OK ==
tiledb_consolidation_plan_create_with_mbr(
ctx_c_, array.ptr().get(), 1024 * 1024, &consolidation_plan));

// Check dump
char* str = nullptr;
tiledb_consolidation_plan_dump_json_str(
ctx_.ptr().get(), consolidation_plan, &str);
tiledb_consolidation_plan_dump_json_str(ctx_c_, consolidation_plan, &str);

std::string plan(str);
CHECK(plan == "{\n \"nodes\": [\n ]\n}\n");
Expand Down
Loading

0 comments on commit e0bc0dd

Please sign in to comment.