diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c08e4c2dd..e05064762d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1222,6 +1222,12 @@ else() set(KRB5_REALM_OVERRIDE -Wl,-U,krb5_realm_override_loaded krb5_realm_override) endif() +## yaml +find_package(Yaml REQUIRED) +include_directories(SYSTEM ${YAML_INCLUDE_DIR}) +ADD_THIRDPARTY_LIB(yaml + STATIC_LIB "${YAML_STATIC_LIB}") + ## Boost # We use a custom cmake module and not cmake's FindBoost. diff --git a/cmake_modules/FindYaml.cmake b/cmake_modules/FindYaml.cmake new file mode 100644 index 0000000000..f7def1bc2c --- /dev/null +++ b/cmake_modules/FindYaml.cmake @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +find_path(YAML_INCLUDE_DIR yaml-cpp/yaml.h + # make sure we don't accidentally pick up a different version + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) +find_library(YAML_STATIC_LIB libyaml-cpp.a + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(YAML REQUIRED_VARS + YAML_STATIC_LIB YAML_INCLUDE_DIR) diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt index f12d181e8e..47334c63b2 100644 --- a/src/kudu/util/CMakeLists.txt +++ b/src/kudu/util/CMakeLists.txt @@ -230,6 +230,7 @@ set(UTIL_SRCS version_info.cc version_util.cc website_util.cc + yamlreader.cc zlib.cc ) @@ -257,6 +258,7 @@ set(UTIL_LIBS pb_util_proto protobuf version_info_proto + yaml zlib) if(NOT APPLE) @@ -293,7 +295,6 @@ set(UTIL_COMPRESSION_SRCS set(UTIL_COMPRESSION_LIBS kudu_util util_compression_proto - glog gutil lz4 @@ -467,6 +468,7 @@ ADD_KUDU_TEST(ttl_cache-test) ADD_KUDU_TEST(url-coding-test) ADD_KUDU_TEST(user-test) ADD_KUDU_TEST(version_util-test) +ADD_KUDU_TEST(yamlreader-test) if (NOT APPLE) ADD_KUDU_TEST(minidump-test) diff --git a/src/kudu/util/yamlreader-test.cc b/src/kudu/util/yamlreader-test.cc new file mode 100644 index 0000000000..8394cf4440 --- /dev/null +++ b/src/kudu/util/yamlreader-test.cc @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/util/yamlreader.h" + +#include +#include +#include + +#include +#include +#include + +#include "kudu/util/env.h" +#include "kudu/util/slice.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" + +using std::string; +using std::unique_ptr; +using std::vector; + +namespace kudu { + +class YamlReaderTest : public KuduTest { +public: + Status GenerateYamlReader(const string& content, unique_ptr* result) { + string fname = GetTestPath("YamlReaderTest.json"); + unique_ptr writable_file; + RETURN_NOT_OK(env_->NewWritableFile(fname, &writable_file)); + RETURN_NOT_OK(writable_file->Append(Slice(content))); + RETURN_NOT_OK(writable_file->Close()); + result->reset(new YamlReader(fname)); + return Status::OK(); + } +}; + +TEST_F(YamlReaderTest, FileNotExist) { + YamlReader r("YamlReaderTest.NotExist"); + Status s = r.Init(); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_STR_CONTAINS( + s.ToString(), "YAML::LoadFile error"); +} + +TEST_F(YamlReaderTest, Corruption) { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("foo", &r)); + ASSERT_OK(r->Init()); + int val = 0; + ASSERT_TRUE(YamlReader::ExtractScalar(r->node(), "foo", &val).IsCorruption()); +} + +TEST_F(YamlReaderTest, EmptyFile) { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("", &r)); + ASSERT_OK(r->Init()); + + int val = 0; + ASSERT_TRUE(YamlReader::ExtractScalar(r->node(), "foo", &val).IsCorruption()); +} + +TEST_F(YamlReaderTest, KeyNotExist) { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("foo: 1", &r)); + ASSERT_OK(r->Init()); + + int val = 0; + ASSERT_TRUE(YamlReader::ExtractScalar(r->node(), "bar", &val).IsNotFound()); +} + +TEST_F(YamlReaderTest, Scalar) { + { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("bool_val: false", &r)); + ASSERT_OK(r->Init()); + bool val = true; + ASSERT_OK(YamlReader::ExtractScalar(r->node(), "bool_val", &val)); + ASSERT_EQ(val, false); + } + + { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("int_val: 123", &r)); + ASSERT_OK(r->Init()); + int val = 0; + ASSERT_OK(YamlReader::ExtractScalar(r->node(), "int_val", &val)); + ASSERT_EQ(val, 123); + } + + { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("double_val: 123.456", &r)); + ASSERT_OK(r->Init()); + double val = 0.0; + ASSERT_OK(YamlReader::ExtractScalar(r->node(), "double_val", &val)); + ASSERT_EQ(val, 123.456); + } + + { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("string_val: hello yaml", &r)); + ASSERT_OK(r->Init()); + string val; + ASSERT_OK(YamlReader::ExtractScalar(r->node(), "string_val", &val)); + ASSERT_EQ(val, "hello yaml"); + } +} + +TEST_F(YamlReaderTest, Map) { + unique_ptr r; + ASSERT_OK(GenerateYamlReader( + "map_val: { key1: hello yaml, key2: 123.456 , key3: 123, key4: false}", &r)); + ASSERT_OK(r->Init()); + + YAML::Node node; + ASSERT_OK(YamlReader::ExtractMap(r->node(), "map_val", &node)); + + { + string val; + ASSERT_OK(YamlReader::ExtractScalar(&node, "key1", &val)); + ASSERT_EQ(val, "hello yaml"); + } + + { + double val = 0.0; + ASSERT_OK(YamlReader::ExtractScalar(&node, "key2", &val)); + ASSERT_EQ(val, 123.456); + } + + { + int val = 0; + ASSERT_OK(YamlReader::ExtractScalar(&node, "key3", &val)); + ASSERT_EQ(val, 123); + } + + { + bool val = true; + ASSERT_OK(YamlReader::ExtractScalar(&node, "key4", &val)); + ASSERT_EQ(val, false); + } + + // Not exist key. + { + int val = 0; + ASSERT_TRUE(YamlReader::ExtractScalar(&node, "key5", &val).IsNotFound()); + } +} + +TEST_F(YamlReaderTest, Array) { + unique_ptr r; + ASSERT_OK(GenerateYamlReader("list_val: [1, 3, 5, 7, 9]", &r)); + ASSERT_OK(r->Init()); + const std::vector& expect_vals = { 1, 3, 5, 7, 9 }; + + std::vector vals; + ASSERT_OK(YamlReader::ExtractArray(r->node(), "list_val", &vals)); + ASSERT_EQ(vals, expect_vals); +} + +} // namespace kudu diff --git a/src/kudu/util/yamlreader.cc b/src/kudu/util/yamlreader.cc new file mode 100644 index 0000000000..a3ee8e0d88 --- /dev/null +++ b/src/kudu/util/yamlreader.cc @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/util/yamlreader.h" + +#include +#include + +#include +// IWYU pragma: no_include +// IWYU pragma: no_include + +#include "kudu/gutil/port.h" +#include "kudu/gutil/strings/substitute.h" + +using std::string; +using strings::Substitute; +using YAML::Node; +using YAML::NodeType; + +namespace kudu { + +YamlReader::YamlReader(string filename) : filename_(std::move(filename)) {} + +Status YamlReader::Init() { + try { + node_ = YAML::LoadFile(filename_); + } catch (std::exception& e) { + return Status::Corruption(Substitute("YAML::LoadFile error: $0", e.what())); + } + + return Status::OK(); +} + +Status YamlReader::ExtractMap(const Node* node, + const string& field, + Node* result) { + CHECK(result); + Node val; + RETURN_NOT_OK(ExtractField(node, field, &val)); + if (PREDICT_FALSE(!val.IsMap())) { + return Status::Corruption(Substitute( + "wrong type during field extraction: expected map but got $0", + TypeToString(val.Type()))); + } + *result = val; + return Status::OK(); +} + +Status YamlReader::ExtractField(const Node* node, + const string& field, + Node* result) { + if (PREDICT_FALSE(!node->IsDefined() || !node->IsMap())) { + return Status::Corruption("node is not map type"); + } + try { + *result = (*node)[field]; + } catch (std::exception& e) { + return Status::NotFound(Substitute("parse field $0 error: $1", field, e.what())); + } + if (PREDICT_FALSE(!result->IsDefined())) { + return Status::Corruption("Missing field", field); + } + + return Status::OK(); +} + +const char* YamlReader::TypeToString(NodeType::value t) { + switch (t) { + case NodeType::Undefined: + return "undefined"; + case NodeType::Null: + return "null"; + case NodeType::Scalar: + return "scalar"; + case NodeType::Sequence: + return "sequence"; + case NodeType::Map: + return "map"; + default: + LOG(FATAL) << "unexpected type: " << t; + } + return ""; +} + +} // namespace kudu diff --git a/src/kudu/util/yamlreader.h b/src/kudu/util/yamlreader.h new file mode 100644 index 0000000000..2186c58f31 --- /dev/null +++ b/src/kudu/util/yamlreader.h @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include + +#include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include // IWYU pragma: keep + +#include "kudu/gutil/macros.h" +#include "kudu/gutil/port.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/status.h" + +namespace kudu { + +// Wraps the YAML parsing functionality of YAML::Node. +// +// This class can read yaml content from a file, extract scalar, map and array type of values. +class YamlReader { + public: + explicit YamlReader(std::string filename); + ~YamlReader() = default; + + Status Init(); + + template + static Status ExtractScalar(const YAML::Node* node, + const std::string& field, + T* result); + + static Status ExtractMap(const YAML::Node* node, + const std::string& field, + YAML::Node* result); + + template + static Status ExtractArray(const YAML::Node* node, + const std::string& field, + std::vector* result); + + const YAML::Node* node() const { return &node_; } + + private: + static const char* TypeToString(YAML::NodeType::value t); + + static Status ExtractField(const YAML::Node* node, + const std::string& field, + YAML::Node* result); + + std::string filename_; + YAML::Node node_; + + DISALLOW_COPY_AND_ASSIGN(YamlReader); +}; + +template +Status YamlReader::ExtractScalar(const YAML::Node* node, + const std::string& field, + T* result) { + CHECK(result); + YAML::Node val; + RETURN_NOT_OK(ExtractField(node, field, &val)); + if (PREDICT_FALSE(!val.IsScalar())) { + return Status::Corruption(strings::Substitute( + "wrong type during field extraction: expected scalar but got $0", + TypeToString(val.Type()))); + } + *result = val.as(); + return Status::OK(); +} + +template +Status YamlReader::ExtractArray(const YAML::Node* node, + const std::string& field, + std::vector* result) { + CHECK(result); + YAML::Node val; + RETURN_NOT_OK(ExtractField(node, field, &val)); + if (PREDICT_FALSE(!val.IsSequence())) { + return Status::Corruption(strings::Substitute( + "wrong type during field extraction: expected sequence but got $0", + TypeToString(val.Type()))); + } + result->reserve(val.size()); + for (YAML::const_iterator iter = val.begin(); iter != val.end(); ++iter) { + try { + if (PREDICT_FALSE(!iter->IsScalar())) { + return Status::Corruption(strings::Substitute( + "wrong type during field extraction: expected scalar but got $0", + TypeToString(iter->Type()))); + } + result->push_back(iter->as()); + } catch (std::exception& e) { + return Status::Corruption(strings::Substitute("parse list element error: $0", e.what())); + } + } + return Status::OK(); +} + +} // namespace kudu diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh index 2ae2f59ae5..eeeb3b6302 100644 --- a/thirdparty/build-definitions.sh +++ b/thirdparty/build-definitions.sh @@ -919,3 +919,23 @@ build_bison() { make -j$PARALLEL $EXTRA_MAKEFLAGS install popd } + +build_yaml() { + YAML_BDIR=$TP_BUILD_DIR/$YAML_NAME$MODE_SUFFIX + mkdir -p $YAML_BDIR + pushd $YAML_BDIR + rm -rf CMakeCache.txt CMakeFiles/ + CFLAGS="$EXTRA_CFLAGS" \ + CXXFLAGS="$EXTRA_CXXFLAGS" \ + LDFLAGS="$EXTRA_LDFLAGS" \ + LIBS="$EXTRA_LIBS" \ + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DYAML_CPP_BUILD_TESTS=OFF \ + -DYAML_CPP_BUILD_TOOLS=OFF \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + $EXTRA_CMAKE_FLAGS \ + $YAML_SOURCE + ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install + popd +} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 0fe31ca991..0c2d451896 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -100,6 +100,7 @@ else "hadoop") F_HADOOP=1 ;; "hive") F_HIVE=1 ;; "sentry") F_SENTRY=1 ;; + "yaml") F_YAML=1 ;; *) echo "Unknown module: $arg"; exit 1 ;; esac done @@ -371,6 +372,10 @@ if [ -n "$F_UNINSTRUMENTED" -o -n "$F_THRIFT" ]; then build_thrift fi +if [ -n "$F_UNINSTRUMENTED" -o -n "$F_YAML" ]; then + build_yaml +fi + restore_env # If we're on macOS best to exit here, otherwise single dependency builds will try to @@ -551,6 +556,10 @@ if [ -n "$F_TSAN" -o -n "$F_THRIFT" ]; then build_thrift fi +if [ -n "$F_TSAN" -o -n "$F_YAML" ]; then + build_yaml +fi + restore_env finish diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index a6fa2b948e..7eeccdfd04 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -430,5 +430,11 @@ fetch_and_patch \ $SENTRY_SOURCE \ $SENTRY_PATCHLEVEL +YAML_PATCHLEVEL=0 +fetch_and_patch \ + $YAML_NAME.tar.gz \ + $YAML_SOURCE \ + $YAML_PATCHLEVEL + echo "---------------" echo "Thirdparty dependencies downloaded successfully" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 01eea406dd..df92495aa3 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -234,3 +234,8 @@ HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME SENTRY_VERSION=505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5 SENTRY_NAME=sentry-$SENTRY_VERSION SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME + +YAML_VERSION=0.6.2 +YAML_NAME=yaml-cpp-yaml-cpp-$YAML_VERSION +YAML_SOURCE=$TP_SOURCE_DIR/$YAML_NAME +