From 30fa14443b27c26ad4af1fc73a3223dfd4e08360 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin Date: Thu, 7 Sep 2017 23:17:02 -0700 Subject: [PATCH] Removing some irrelevant files Summary: Removing some irrelevant files (Kudu Python client, some build support tools we are not using, disclaimer, etc.) Test Plan: Build locally Jenkins: skip Reviewers: karthik, kannan, bharat Reviewed By: kannan, bharat Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D2875 --- CONTRIBUTING.adoc | 18 - DISCLAIMER | 8 - Doxyfile | 32 +- LICENSE.txt | 20 +- build-support/check_compatibility.py | 232 ---- build-support/dist_test.py | 442 -------- build-support/generate_precompiled_xxd.sh | 54 - build-support/get-upstream-commit.sh | 38 +- build-support/test_result_server.py | 471 -------- build-support/trigger_gerrit.py | 129 --- python/MANIFEST.in | 26 - python/Makefile | 36 - python/README.md | 12 - python/kudu/__init__.pxd | 17 - python/kudu/__init__.py | 128 --- python/kudu/client.pyx | 1252 --------------------- python/kudu/compat.py | 101 -- python/kudu/errors.pxd | 20 - python/kudu/errors.pyx | 77 -- python/kudu/libkudu_client.pxd | 607 ---------- python/kudu/schema.pxd | 59 - python/kudu/schema.pyx | 560 --------- python/kudu/tests/__init__.py | 0 python/kudu/tests/common.py | 168 --- python/kudu/tests/test_client.py | 204 ---- python/kudu/tests/test_scanner.py | 120 -- python/kudu/tests/test_schema.py | 197 ---- python/kudu/util.py | 36 - python/requirements.txt | 6 - python/setup.cfg | 15 - python/setup.py | 182 --- 31 files changed, 46 insertions(+), 5221 deletions(-) delete mode 100644 CONTRIBUTING.adoc delete mode 100644 DISCLAIMER delete mode 100755 build-support/check_compatibility.py delete mode 100755 build-support/dist_test.py delete mode 100755 build-support/generate_precompiled_xxd.sh delete mode 100755 build-support/test_result_server.py delete mode 100755 build-support/trigger_gerrit.py delete mode 100644 python/MANIFEST.in delete mode 100644 python/Makefile delete mode 100644 python/README.md delete mode 100644 python/kudu/__init__.pxd delete mode 100644 python/kudu/__init__.py delete mode 100644 python/kudu/client.pyx delete mode 100644 python/kudu/compat.py delete mode 100644 python/kudu/errors.pxd delete mode 100644 python/kudu/errors.pyx delete mode 100644 python/kudu/libkudu_client.pxd delete mode 100644 python/kudu/schema.pxd delete mode 100644 python/kudu/schema.pyx delete mode 100644 python/kudu/tests/__init__.py delete mode 100644 python/kudu/tests/common.py delete mode 100644 python/kudu/tests/test_client.py delete mode 100644 python/kudu/tests/test_scanner.py delete mode 100644 python/kudu/tests/test_schema.py delete mode 100644 python/kudu/util.py delete mode 100644 python/requirements.txt delete mode 100644 python/setup.cfg delete mode 100644 python/setup.py diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc deleted file mode 100644 index 5bd92860e1e6..000000000000 --- a/CONTRIBUTING.adoc +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Please read the instructions for http://getkudu.io/docs/contributing.html[how to contribute to Kudu]. diff --git a/DISCLAIMER b/DISCLAIMER deleted file mode 100644 index 2fc823b090ec..000000000000 --- a/DISCLAIMER +++ /dev/null @@ -1,8 +0,0 @@ -Apache Kudu is an effort undergoing incubation at The Apache Software -Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is -required of all newly accepted projects until a further review -indicates that the infrastructure, communications, and decision making -process have stabilized in a manner consistent with other successful -ASF projects. While incubation status is not necessarily a reflection -of the completeness or stability of the code, it does indicate that -the project has yet to be fully endorsed by the ASF. diff --git a/Doxyfile b/Doxyfile index 62218fa4584e..9817562c4bb8 100644 --- a/Doxyfile +++ b/Doxyfile @@ -1,5 +1,22 @@ # Doxyfile 1.8.5 +# This file was initially automatically generated by Doxygen. +# +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# + # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. # @@ -24,21 +41,6 @@ # for the list of possible encodings. # The default value is: UTF-8. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by diff --git a/LICENSE.txt b/LICENSE.txt index 467d00dcbdaf..105dfec6f76c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -201,7 +201,7 @@ limitations under the License. -------------------------------------------------------------------------------- -src/kudu/gutil (some portions): Apache 2.0, and 3-clause BSD +src/yb/gutil (some portions): Apache 2.0, and 3-clause BSD This module is derived from code in the Chromium project, copyright (c) Google inc and (c) The Chromium Authors and licensed under the @@ -237,7 +237,7 @@ Apache 2.0 License or the under the 3-clause BSD license: -------------------------------------------------------------------------------- -src/kudu/gutil/utf: licensed under the following terms: +src/yb/gutil/utf: licensed under the following terms: UTF-8 Library @@ -256,7 +256,7 @@ src/kudu/gutil/utf: licensed under the following terms: -------------------------------------------------------------------------------- -src/kudu/gutil/valgrind.h: Hybrid BSD (half BSD, half zlib) +src/yb/gutil/valgrind.h: Hybrid BSD (half BSD, half zlib) This file is part of Valgrind, a dynamic binary instrumentation framework. @@ -295,7 +295,7 @@ src/kudu/gutil/valgrind.h: Hybrid BSD (half BSD, half zlib) SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -src/kudu/util (some portions): 3-clause BSD license +src/yb/util (some portions): 3-clause BSD license Some portions of this module are derived from code from LevelDB ( https://github.com/google/leveldb ): @@ -329,7 +329,7 @@ Some portions of this module are derived from code from LevelDB OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -src/kudu/util (HdrHistogram-related classes): public domain +src/yb/util (HdrHistogram-related classes): public domain Portions of these classes were ported from Java to C++ from the sources available at https://github.com/HdrHistogram/HdrHistogram . @@ -340,7 +340,7 @@ available at https://github.com/HdrHistogram/HdrHistogram . -------------------------------------------------------------------------------- -src/kudu/util/random-util.cc: some portions adapted from WebRTC project +src/yb/util/random-util.cc: some portions adapted from WebRTC project (modules/video_coding/main/test/test_util.cc) under a 3-clause BSD license. Copyright (c) 2011, The WebRTC project authors. All rights reserved. @@ -375,7 +375,7 @@ src/kudu/util/random-util.cc: some portions adapted from WebRTC project -------------------------------------------------------------------------------- -src/kudu/util/sync_point: adapted from the RocksDB project under a 3-clause +src/yb/util/sync_point: adapted from the RocksDB project under a 3-clause BSD license with an additional grant of patent rights: Copyright (c) 2014, Facebook, Inc. @@ -445,7 +445,7 @@ BSD license with an additional grant of patent rights: -------------------------------------------------------------------------------- -src/kudu/server/url-coding.cc: some portions adapted from the Boost project +src/yb/server/url-coding.cc: some portions adapted from the Boost project thirdparty/boost_uuid/: Boost Software License - Version 1.0 - August 17th, 2003 @@ -576,7 +576,7 @@ www/jquery*.js: MIT license -------------------------------------------------------------------------------- -java/kudu-client/src/main/java/org/kududb/util/: Slice.java and Slices.java +java/yb-client/src/main/java/org/ybdb/util/: Slice.java and Slices.java Derived from the LevelDB Java project at https://github.com/dain/leveldb/ Licensed under the Apache 2.0 license with the following copyrights: @@ -598,7 +598,7 @@ java/kudu-client/src/main/java/org/kududb/util/: Slice.java and Slices.java -------------------------------------------------------------------------------- -java/kudu-client/: Some classes are derived from the AsyncHBase project +java/yb-client/: Some classes are derived from the AsyncHBase project under the following 3-clause BSD license: Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved. diff --git a/build-support/check_compatibility.py b/build-support/check_compatibility.py deleted file mode 100755 index f2e5ac89aae0..000000000000 --- a/build-support/check_compatibility.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -# Script which checks Java API compatibility between two revisions of the -# Java client. -# -# Based on the compatibility checker from the HBase project, but ported to -# Python for better readability. - -import logging -import optparse -import os -import re -import shutil -import subprocess -import sys - -from yb_util import check_output - -JAVA_ACC_GIT_URL = "https://github.com/lvc/japi-compliance-checker.git" - -# The annotations for what we consider our public API. -PUBLIC_ANNOTATIONS = ["InterfaceAudience.LimitedPrivate", - "InterfaceAudience.Public"] - -# Various relative paths -PATH_TO_REPO_DIR = "../" -PATH_TO_BUILD_DIR = "../build/compat-check" - - -def get_repo_dir(): - """ Return the path to the top of the repo. """ - dirname, _ = os.path.split(os.path.abspath(__file__)) - return os.path.abspath(os.path.join(dirname, PATH_TO_REPO_DIR)) - - -def get_scratch_dir(): - """ Return the path to the scratch dir that we build within. """ - dirname, _ = os.path.split(os.path.abspath(__file__)) - return os.path.abspath(os.path.join(dirname, PATH_TO_BUILD_DIR)) - - -def get_java_acc_dir(): - """ Return the path where we check out the Java API Compliance Checker. """ - return os.path.join(get_repo_dir(), "thirdparty/java-acc") - - -def clean_scratch_dir(scratch_dir): - """ Clean up and re-create the scratch directory. """ - if os.path.exists(scratch_dir): - logging.info("Removing scratch dir %s...", scratch_dir) - shutil.rmtree(scratch_dir) - logging.info("Creating empty scratch dir %s...", scratch_dir) - os.makedirs(scratch_dir) - - -def checkout_java_tree(rev, path): - """ Check out the Java source tree for the given revision into the given path. """ - logging.info("Checking out %s in %s", rev, path) - os.makedirs(path) - # Extract java source - subprocess.check_call(["bash", '-o', 'pipefail', "-c", - ("git archive --format=tar %s java/ | " + - "tar -C \"%s\" -xf -") % (rev, path)], - cwd=get_repo_dir()) - # Extract proto files which the Java build also relies on. - subprocess.check_call(["bash", '-o', 'pipefail', "-c", - ("git archive --format=tar %s src/ | " + - "tar -C \"%s\" --wildcards -xf - '*.proto'") % (rev, path)], - cwd=get_repo_dir()) - # Symlink thirdparty from the outer build so that protoc is available. - # This may break at some point in the future if we switch protobuf versions, - # but for now it's faster than rebuilding protobuf in both trees. - os.symlink(os.path.join(get_repo_dir(), "thirdparty"), - os.path.join(path, "thirdparty")) - - -def get_git_hash(revname): - """ Convert 'revname' to its SHA-1 hash. """ - return check_output(["git", "rev-parse", revname], - cwd=get_repo_dir()).strip() - - -def build_tree(path): - """ Run the Java build within 'path'. """ - java_path = os.path.join(path, "java") - logging.info("Building in %s...", java_path) - subprocess.check_call(["mvn", "-DskipTests", "-Dmaven.javadoc.skip=true", - "package"], - cwd=java_path) - - -def checkout_java_acc(force): - """ - Check out the Java API Compliance Checker. If 'force' is true, will re-download even if the - directory exists. - """ - acc_dir = get_java_acc_dir() - if os.path.exists(acc_dir): - logging.info("Java JAVA_ACC is already downloaded.") - if not force: - return - logging.info("Forcing re-download.") - shutil.rmtree(acc_dir) - logging.info("Checking out Java JAVA_ACC...") - subprocess.check_call(["git", "clone", "--depth=1", JAVA_ACC_GIT_URL, acc_dir]) - - -def find_client_jars(path): - """ Return a list of jars within 'path' to be checked for compatibility. """ - all_jars = set(check_output(["find", path, "-name", "*.jar"]).splitlines()) - - # If we see "original-foo.jar", then remove "foo.jar" since that's a post-shading - # duplicate. - dups = [] - for j in all_jars: - dirname, name = os.path.split(j) - m = re.match("original-(.+)", name) - if m: - dups.append(os.path.join(dirname, m.group(1))) - for d in dups: - all_jars.remove(d) - - return [j for j in all_jars if ( - "-tests" not in j and - "-sources" not in j and - "-with-dependencies" not in j)] - - -def run_java_acc(src_name, src, dst_name, dst): - """ Run the compliance checker to compare 'src' and 'dst'. """ - src_jars = find_client_jars(src) - dst_jars = find_client_jars(dst) - logging.info("Will check compatibility between original jars:\n%s\n" + - "and new jars:\n%s", - "\n".join(src_jars), - "\n".join(dst_jars)) - - annotations_path = os.path.join(get_scratch_dir(), "annotations.txt") - with file(annotations_path, "w") as f: - for ann in PUBLIC_ANNOTATIONS: - print >>f, ann - - java_acc_path = os.path.join(get_java_acc_dir(), "japi-compliance-checker.pl") - - out_path = os.path.join(get_scratch_dir(), "report.html") - subprocess.check_call(["perl", java_acc_path, - "-l", "Kudu", - "-v1", src_name, - "-v2", dst_name, - "-d1", ",".join(src_jars), - "-d2", ",".join(dst_jars), - "-report-path", out_path, - "-annotations-list", annotations_path]) - - -def main(argv): - logging.basicConfig(level=logging.INFO) - parser = optparse.OptionParser( - usage="usage: %prog SRC..[DST]") - parser.add_option("-f", "--force-download", dest="force_download_deps", - help=("Download dependencies (i.e. Java JAVA_ACC) even if they are " + - "already present")) - opts, args = parser.parse_args() - - if len(args) != 1: - parser.error("no src/dst revision specified") - sys.exit(1) - - src_rev, dst_rev = args[0].split("..", 1) - if dst_rev == "": - dst_rev = "HEAD" - src_rev = get_git_hash(src_rev) - dst_rev = get_git_hash(dst_rev) - - logging.info("Source revision: %s", src_rev) - logging.info("Destination revision: %s", dst_rev) - - # Download deps. - checkout_java_acc(opts.force_download_deps) - - # Set up the build. - scratch_dir = get_scratch_dir() - clean_scratch_dir(scratch_dir) - - # Check out the src and dst source trees. - src_dir = os.path.join(scratch_dir, "src") - dst_dir = os.path.join(scratch_dir, "dst") - checkout_java_tree(src_rev, src_dir) - checkout_java_tree(dst_rev, dst_dir) - - # Run the build in each. - build_tree(src_dir) - build_tree(dst_dir) - - run_java_acc(src_rev, src_dir, - dst_rev, dst_dir) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/build-support/dist_test.py b/build-support/dist_test.py deleted file mode 100755 index 6a5174321e98..000000000000 --- a/build-support/dist_test.py +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env python2 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -# This tool allows tests to be submitted to a distributed testing -# service running on shared infrastructure. -# -# See dist_test.py --help for usage information. - -import argparse -import glob -try: - import simplejson as json -except: - import json -import logging -import os -import pprint -import re -import sys -import shlex -import shutil -import subprocess -import time - -TEST_TIMEOUT_SECS = int(os.environ.get('TEST_TIMEOUT_SECS', '900')) -ARTIFACT_ARCHIVE_GLOBS = ["build/*/test-logs/*"] -ISOLATE_SERVER = os.environ.get('ISOLATE_SERVER', - "http://isolate.cloudera.org:4242/") -DIST_TEST_HOME = os.environ.get('DIST_TEST_HOME', - os.path.expanduser("~/dist_test")) - -# The number of times that flaky tests will be retried. -# Our non-distributed implementation sets a number of _attempts_, not a number -# of retries, so we have to subtract 1. -FLAKY_TEST_RETRIES = int(os.environ.get('YB_FLAKY_TEST_ATTEMPTS', 1)) - 1 - -PATH_TO_REPO = "../" - -TEST_COMMAND_RE = re.compile('Test command: (.+)$') -LDD_RE = re.compile(r'^\s+.+? => (\S+) \(0x.+\)') - -DEPS_FOR_ALL = \ - ["build-support/stacktrace_addr2line.pl", - "build-support/run-test.sh", - "build-support/run_dist_test.py", - "build-support/tsan-suppressions.txt", - "build-support/lsan-suppressions.txt", - - # The LLVM symbolizer is necessary for suppressions to work - "thirdparty/installed/bin/llvm-symbolizer", - - # Tests that use the external minicluster require these. - # TODO: declare these dependencies per-test. - "build/latest/bin/yb-tserver", - "build/latest/bin/yb-master", - "build/latest/bin/yb-ts-cli", - - # parser-test requires these data files. - # TODO: again, we should do this with some per-test metadata file. - # TODO: these are broken now that we separate source and build trees. - #".../example-deletes.txt", - #".../example-tweets.txt", - - # Tests that require tooling require these. - "build/latest/bin/yb-admin", - ] - - -class StagingDir(object): - @staticmethod - def new(): - dir = rel_to_abs("build/isolate") - if os.path.isdir(dir): - shutil.rmtree(dir) - os.makedirs(dir) - return StagingDir(dir) - - def __init__(self, dir): - self.dir = dir - - def archive_dump_path(self): - return os.path.join(self.dir, "dump.json") - - def gen_json_paths(self): - return glob.glob(os.path.join(self.dir, "*.gen.json")) - - def tasks_json_path(self): - return os.path.join(self.dir, "tasks.json") - - -def rel_to_abs(rel_path): - dirname, _ = os.path.split(os.path.abspath(__file__)) - abs = os.path.abspath(os.path.join(dirname, PATH_TO_REPO, rel_path)) - if rel_path.endswith('/') and not abs.endswith('/'): - abs += '/' - return abs - - -def abs_to_rel(abs_path, staging): - rel = os.path.relpath(abs_path, staging.dir) - if abs_path.endswith('/') and not rel.endswith('/'): - rel += '/' - return rel - - -def get_test_commandlines(): - ctest_bin = os.path.join(rel_to_abs("thirdparty/installed/bin/ctest")) - p = subprocess.Popen([ctest_bin, "-V", "-N", "-LE", "no_dist_test"], stdout=subprocess.PIPE) - out, err = p.communicate() - if p.returncode != 0: - print >>sys.stderr, "Unable to list tests with ctest" - sys.exit(1) - lines = out.splitlines() - commands = [] - for l in lines: - m = TEST_COMMAND_RE.search(l) - if not m: - continue - commands.append(shlex.split(m.group(1))) - return commands - - -def is_lib_blacklisted(lib): - # These particular system libraries, we should ship to the remote nodes. - # No need to ship things like libc, libstdcxx, etc. - if "boost" in lib or "oauth" in lib: - return False - if lib.startswith("/lib") or lib.startswith("/usr"): - return True - return False - - -def is_outside_of_tree(path): - repo_dir = rel_to_abs("./") - rel = os.path.relpath(path, repo_dir) - return rel.startswith("../") - -def copy_system_library(lib): - """ - For most system libraries, we expect them to be installed on the test - machines. However, a couple are shipped from the submitter machine - to the cluster by putting them in a special directory inside the - isolated build tree. - - This function copies such libraries into that directory. - """ - sys_lib_dir = rel_to_abs("build/dist-test-system-libs") - if not os.path.exists(sys_lib_dir): - os.makedirs(sys_lib_dir) - dst = os.path.join(sys_lib_dir, os.path.basename(lib)) - if not os.path.exists(dst): - logging.info("Copying system library %s to %s...", lib, dst) - shutil.copy2(rel_to_abs(lib), dst) - return dst - - -def ldd_deps(exe): - """ - Runs 'ldd' on the provided 'exe' path, returning a list of - any libraries it depends on. Blacklisted libraries are - removed from this list. - - If the provided 'exe' is not a binary executable, returns - an empty list. - """ - if exe.endswith(".sh"): - return [] - p = subprocess.Popen(["ldd", exe], stdout=subprocess.PIPE) - out, err = p.communicate() - if p.returncode != 0: - print >>sys.stderr, "failed to run ldd on ", exe - return [] - ret = [] - for l in out.splitlines(): - m = LDD_RE.match(l) - if not m: - continue - lib = m.group(1) - if is_lib_blacklisted(lib): - continue - path = m.group(1) - ret.append(m.group(1)) - - # ldd will often point to symlinks. We need to upload the symlink - # as well as whatever it's pointing to, recursively. - while os.path.islink(path): - path = os.path.join(os.path.dirname(path), os.readlink(path)) - ret.append(path) - return ret - - -def num_shards_for_test(test_name): - if 'raft_consensus-itest' in test_name: - return 8 - if 'cfile-test' in test_name: - return 4 - if 'mt-tablet-test' in test_name: - return 4 - return 1 - - -def create_archive_input(staging, argv, - disable_sharding=False): - """ - Generates .gen.json and .isolate files corresponding to the - test command 'argv'. The outputs are placed in the specified - staging directory. - - Some larger tests are automatically sharded into several tasks. - If 'disable_sharding' is True, this behavior will be suppressed. - """ - if not argv[0].endswith('run-test.sh') or len(argv) < 2: - print >>sys.stderr, "Unable to handle test: ", argv - return - test_name = os.path.basename(argv[1]) - abs_test_exe = os.path.realpath(argv[1]) - rel_test_exe = abs_to_rel(abs_test_exe, staging) - argv[1] = rel_test_exe - files = [] - files.append(rel_test_exe) - deps = ldd_deps(abs_test_exe) - for d in DEPS_FOR_ALL: - d = os.path.realpath(rel_to_abs(d)) - if os.path.isdir(d): - d += "/" - deps.append(d) - for d in deps: - # System libraries will end up being relative paths out - # of the build tree. We need to copy those into the build - # tree somewhere. - if is_outside_of_tree(d): - d = copy_system_library(d) - files.append(abs_to_rel(d, staging)) - - if disable_sharding: - num_shards = 1 - else: - num_shards = num_shards_for_test(test_name) - for shard in xrange(0, num_shards): - out_archive = os.path.join(staging.dir, '%s.%d.gen.json' % (test_name, shard)) - out_isolate = os.path.join(staging.dir, '%s.%d.isolate' % (test_name, shard)) - - command = ['../../build-support/run_dist_test.py', - '-e', 'GTEST_SHARD_INDEX=%d' % shard, - '-e', 'GTEST_TOTAL_SHARDS=%d' % num_shards, - '-e', 'YB_TEST_TIMEOUT=%d' % (TEST_TIMEOUT_SECS - 30), - '-e', 'YB_ALLOW_SLOW_TESTS=%s' % os.environ.get('YB_ALLOW_SLOW_TESTS', 1), - '-e', 'YB_COMPRESS_TEST_OUTPUT=%s' % \ - os.environ.get('YB_COMPRESS_TEST_OUTPUT', 0)] - command.append('--') - command += argv[1:] - - archive_json = dict(args=["-i", out_isolate, - "-s", out_isolate + "d"], - dir=rel_to_abs("."), - name='%s.%d/%d' % (test_name, shard + 1, num_shards), - version=1) - isolate_dict = dict(variables=dict(command=command, - files=files)) - with open(out_archive, "w") as f: - json.dump(archive_json, f) - with open(out_isolate, "w") as f: - pprint.pprint(isolate_dict, f) - - -def create_task_json(staging, - replicate_tasks=1, - flaky_test_set=set()): - """ - Create a task JSON file suitable for submitting to the distributed - test execution service. - - If 'replicate_tasks' is higher than one, each .isolate file will be - submitted multiple times. This can be useful for looping tests. - """ - tasks = [] - with file(staging.archive_dump_path(), "r") as isolate_dump: - inmap = json.load(isolate_dump) - - # Some versions of 'isolate batcharchive' directly list the items in - # the dumped JSON. Others list it in an 'items' dictionary. - items = inmap.get('items', inmap) - for k, v in items.iteritems(): - # The key is 'foo-test.'. So, chop off the last component - # to get the test name - test_name = ".".join(k.split(".")[:-1]) - max_retries = 0 - if test_name in flaky_test_set: - max_retries = FLAKY_TEST_RETRIES - - tasks += [{"isolate_hash": str(v), - "description": str(k), - "artifact_archive_globs": ARTIFACT_ARCHIVE_GLOBS, - "timeout": TEST_TIMEOUT_SECS + 30, - "max_retries": max_retries - }] * replicate_tasks - - outmap = {"tasks": tasks} - - with file(staging.tasks_json_path(), "wt") as f: - json.dump(outmap, f) - - -def run_isolate(staging): - """ - Runs 'isolate batcharchive' to archive all of the .gen.json files in - the provided staging directory. - - Throws an exception if the call fails. - """ - isolate_path = "isolate" - try: - subprocess.check_call([isolate_path, - 'batcharchive', - '-isolate-server=' + ISOLATE_SERVER, - '-dump-json=' + staging.archive_dump_path(), - '--'] + staging.gen_json_paths()) - except: - print >>sys.stderr, "Failed to run", isolate_path - raise - -def submit_tasks(staging, options): - """ - Runs the distributed testing tool to submit the tasks in the - provided staging directory. - - This requires that the tasks JSON file has already been generated - by 'create_task_json()'. - """ - if not os.path.exists(DIST_TEST_HOME): - print >>sys.stderr, "Cannot find dist_test tools at path %s " \ - "Set the DIST_TEST_HOME environment variable to the path to the dist_test directory. " \ - % DIST_TEST_HOME, - raise OSError("Cannot find path to dist_test tools") - client_py_path = os.path.join(DIST_TEST_HOME, "client.py") - try: - cmd = [client_py_path, "submit"] - if options.no_wait: - cmd.append('--no-wait') - cmd.append(staging.tasks_json_path()) - subprocess.check_call(cmd) - except: - print >>sys.stderr, "Failed to run", client_py_path - raise - -def get_flakies(): - path = os.getenv('YB_FLAKY_TEST_LIST') - if not path: - return set() - return set(l.strip() for l in file(path)) - -def run_all_tests(parser, options): - """ - Gets all of the test command lines from 'ctest', isolates them, - creates a task list, and submits the tasks to the testing service. - """ - commands = get_test_commandlines() - staging = StagingDir.new() - for command in commands: - create_archive_input(staging, command, - disable_sharding=options.disable_sharding) - - run_isolate(staging) - create_task_json(staging, flaky_test_set=get_flakies()) - submit_tasks(staging, options) - -def add_run_all_subparser(subparsers): - p = subparsers.add_parser('run-all', help='Run all of the dist-test-enabled tests') - p.set_defaults(func=run_all_tests) - -def loop_test(parser, options): - """ - Runs many instances of a user-provided test case on the testing service. - """ - if options.num_instances < 1: - parser.error("--num-instances must be >= 1") - command = ["run-test.sh", options.cmd] + options.args - staging = StagingDir.new() - create_archive_input(staging, command, - disable_sharding=options.disable_sharding) - run_isolate(staging) - create_task_json(staging, options.num_instances) - submit_tasks(staging, options) - -def add_loop_test_subparser(subparsers): - p = subparsers.add_parser('loop', help='Run many instances of the same test', - epilog="if passing arguments to the test, you may want to use a '--' " + - "argument before . e.g: loop -- foo-test --gtest_opt=123") - p.add_argument("--num-instances", "-n", dest="num_instances", type=int, - help="number of test instances to start", metavar="NUM", - default=100) - p.add_argument("cmd", help="test binary") - p.add_argument("args", nargs=argparse.REMAINDER, help="test arguments") - p.set_defaults(func=loop_test) - - -def main(argv): - logging.basicConfig(level=logging.INFO) - p = argparse.ArgumentParser() - p.add_argument("--disable-sharding", dest="disable_sharding", action="store_true", - help="Disable automatic sharding of tests", default=False) - p.add_argument("--no-wait", dest="no_wait", action="store_true", - help="Return without waiting for the job to complete", default=False) - sp = p.add_subparsers() - add_loop_test_subparser(sp) - add_run_all_subparser(sp) - args = p.parse_args(argv) - args.func(p, args) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/build-support/generate_precompiled_xxd.sh b/build-support/generate_precompiled_xxd.sh deleted file mode 100755 index 895686548831..000000000000 --- a/build-support/generate_precompiled_xxd.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -e -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -# Script which embeds a piece of raw data into a C++ source file. Functionally -# the same as xxd -i, but inserts custom namespace and variable names. - -IN_FILE=$1 -OUT_FILE=$2 - -echo "// header generated by build-support/generate_precompiled_xxd.sh" > $OUT_FILE -echo "namespace yb {" >> $OUT_FILE -echo "namespace codegen {" >> $OUT_FILE - -echo "extern const char precompiled_ll_data[] = {" >> $OUT_FILE -xxd -i - < $IN_FILE >> $OUT_FILE -# LLVM requires the binary to be null terminated. -echo ", 0x00" >> $OUT_FILE -echo "};" >> $OUT_FILE - -LEN=$(wc -c $IN_FILE | awk '{print $1}') -echo "extern const unsigned int precompiled_ll_len = ${LEN};" >> $OUT_FILE - -echo "} // namespace codegen" >> $OUT_FILE -echo "} // namespace yb" >> $OUT_FILE diff --git a/build-support/get-upstream-commit.sh b/build-support/get-upstream-commit.sh index 7bc3373eaf46..9cc75d522854 100755 --- a/build-support/get-upstream-commit.sh +++ b/build-support/get-upstream-commit.sh @@ -1,18 +1,5 @@ #!/bin/bash # -# Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -30,13 +17,26 @@ # specific language governing permissions and limitations # under the License. # -# Script which tries to determine the most recent git hash in the current -# branch which was checked in using phabricator. This commit hash is printed to -# stdout. +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +# Script which tries to determine the most recent git hash in the current branch which was checked +# in using Phabricator. This commit hash is printed to stdout. # -# It does so by looking for the 'Reviewed By:' tag added by phabricator. This is -# more foolproof than trying to guess at the "origin/" branch name, since the -# developer might be working on some local topic branch. +# It does so by looking for the 'Reviewed By:' tag added by phabricator. This is more foolproof than +# trying to guess at the "origin/" branch name, since the developer might be working on some local +# topic branch. set -e git log --grep='Reviewed By:' -n1 --pretty=format:%H diff --git a/build-support/test_result_server.py b/build-support/test_result_server.py deleted file mode 100755 index 3e40f1396919..000000000000 --- a/build-support/test_result_server.py +++ /dev/null @@ -1,471 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -# Simple HTTP server which receives test results from the build slaves and -# stores them in a MySQL database. The test logs are also stored in an S3 bucket. -# -# Configuration here is done via environment variables: -# -# MySQL config: -# MYSQLHOST - host running mysql -# MYSQLUSER - username -# MYSQLPWD - password -# MYSQLDB - mysql database -# -# S3 config: -# AWS_ACCESS_KEY - AWS access key -# AWS_SECRET_KEY - AWS secret key -# TEST_RESULT_BUCKET - bucket to store results in (eg 'kudu-test-results') -# -# If the AWS credentials are not configured, falls back to using Boto's -# default configuration (http://boto.cloudhackers.com/en/latest/boto_config_tut.html) -# -# Installation instructions: -# You probably want to run this inside a virtualenv to avoid having -# to install python modules systemwide. For example: -# $ virtualenv ~/flaky-test-server-env/ -# $ . ~/flaky-test-server-env/bin/activate -# $ pip install boto -# $ pip install jinja2 -# $ pip install cherrypy -# $ pip install MySQL-python - -import boto -import cherrypy -import gzip -import itertools -from jinja2 import Template -import logging -import MySQLdb -import os -import parse_test_failure -from StringIO import StringIO -import threading -import uuid - -class TRServer(object): - def __init__(self): - self.thread_local = threading.local() - self.ensure_table() - self.s3 = self.connect_s3() - self.s3_bucket = self.s3.get_bucket(os.environ["TEST_RESULT_BUCKET"]) - - def connect_s3(self): - access_key = os.environ.get("AWS_ACCESS_KEY") - secret_key = os.environ.get("AWS_SECRET_KEY") - s3 = boto.connect_s3(access_key, secret_key) - logging.info("Connected to S3 with access key %s" % access_key) - return s3 - - - def upload_to_s3(self, key, fp, filename): - k = boto.s3.key.Key(self.s3_bucket) - k.key = key - # The Content-Disposition header sets the filename that the browser - # will use to download this. - # We have to cast to str() here, because boto will try to escape the header - # incorrectly if you pass a unicode string. - k.set_metadata('Content-Disposition', str('inline; filename=%s' % filename)) - k.set_contents_from_string(fp.read(), - reduced_redundancy=True) - - def connect_mysql(self): - if hasattr(self.thread_local, "db") and \ - self.thread_local.db is not None: - return self.thread_local.db - - host = os.environ["MYSQLHOST"] - user = os.environ["MYSQLUSER"] - pwd = os.environ["MYSQLPWD"] - db = os.environ["MYSQLDB"] - self.thread_local.db = MySQLdb.connect(host, user, pwd, db) - self.thread_local.db.autocommit(True) - logging.info("Connected to MySQL at %s" % host) - return self.thread_local.db - - def execute_query(self, query, *args): - """ Execute a query, automatically reconnecting on disconnection. """ - # We'll try up to 3 times to reconnect - MAX_ATTEMPTS = 3 - - # Error code for the "MySQL server has gone away" error. - MYSQL_SERVER_GONE_AWAY = 2006 - - attempt_num = 0 - while True: - c = self.connect_mysql().cursor(MySQLdb.cursors.DictCursor) - attempt_num = attempt_num + 1 - try: - c.execute(query, *args) - return c - except MySQLdb.OperationalError as err: - if err.args[0] == MYSQL_SERVER_GONE_AWAY and attempt_num < MAX_ATTEMPTS: - logging.warn("Forcing reconnect to MySQL: %s" % err) - self.thread_local.db = None - continue - else: - raise - - - def ensure_table(self): - c = self.execute_query(""" - CREATE TABLE IF NOT EXISTS test_results ( - id int not null auto_increment primary key, - timestamp timestamp not null default current_timestamp, - build_id varchar(100), - revision varchar(50), - build_config varchar(100), - hostname varchar(255), - test_name varchar(100), - status int, - log_key char(40), - INDEX (revision), - INDEX (test_name), - INDEX (timestamp) - );""") - - @cherrypy.expose - def index(self): - return "Welcome to the test result server!" - - @cherrypy.expose - def add_result(self, **kwargs): - args = {} - args.update(kwargs) - - # Only upload the log if it's provided. - if 'log' in kwargs: - log = kwargs['log'] - s3_id = uuid.uuid1() - self.upload_to_s3(s3_id, log.file, log.filename) - else: - s3_id = None - args['log_key'] = s3_id - - logging.info("Handling report: %s" % repr(args)) - - self.execute_query( - "INSERT INTO test_results(build_id, revision, build_config, hostname, test_name, status, log_key) " - "VALUES (%(build_id)s, %(revision)s, %(build_config)s, %(hostname)s, %(test_name)s," - "%(status)s, %(log_key)s)", - args) - return "Success!\n" - - @cherrypy.expose - def download_log(self, key): - expiry = 60 * 60 * 24 # link should last 1 day - k = boto.s3.key.Key(self.s3_bucket) - k.key = key - raise cherrypy.HTTPRedirect(k.generate_url(expiry)) - - @cherrypy.expose - def diagnose(self, key): - k = boto.s3.key.Key(self.s3_bucket) - k.key = key - log_text_gz = k.get_contents_as_string() - log_text = gzip.GzipFile(fileobj=StringIO(log_text_gz)).read().decode('utf-8') - summary = parse_test_failure.extract_failure_summary(log_text) - if not summary: - summary = "Unable to diagnose" - template = Template(""" -

Diagnosed failure

- 
{{ summary|e }}
 -

Full log

- 
{{ log_text|e }}
 - """) - return self.render_container(template.render(summary=summary, log_text=log_text)) - - def recently_failed_html(self): - """ Return an HTML report of recently failed tests """ - c = self.execute_query( - "SELECT * from test_results WHERE status != 0 " - "AND timestamp > NOW() - INTERVAL 1 WEEK " - "ORDER BY timestamp DESC LIMIT 50") - failed_tests = c.fetchall() - - prev_date = None - for t in failed_tests: - t['is_new_date'] = t['timestamp'].date() != prev_date - prev_date = t['timestamp'].date() - - template = Template(""" -

50 most recent failures

- - - - - - - - - - - {% for run in failed_tests %} - {% if run.is_new_date %} - - - - {% endif %} - - - - - - - - - - {% endfor %} -
testconfigexit coderevmachinetimebuild
{{ run.timestamp.date()|e }}
- {{ run.test_name |e }} - {{ run.build_config |e }}{{ run.status |e }} - {% if run.log_key %} - failure log | - diagnose - {% endif %} - {{ run.revision |e }}{{ run.hostname |e }}{{ run.timestamp |e }}{{ run.build_id |e }}
- """) - return template.render(failed_tests=failed_tests) - - def flaky_report_html(self): - """ Return an HTML report of recently flaky tests """ - c = self.execute_query( - """SELECT - test_name, - DATEDIFF(NOW(), timestamp) AS days_ago, - SUM(IF(status != 0, 1, 0)) AS num_failures, - COUNT(*) AS num_runs - FROM test_results - WHERE timestamp > NOW() - INTERVAL 1 WEEK - GROUP BY test_name, days_ago - HAVING num_failures > 0 - ORDER BY test_name""") - rows = c.fetchall() - - results = [] - for test_name, test_rows in itertools.groupby(rows, lambda r: r['test_name']): - # Convert to list so we can consume it multiple times - test_rows = list(test_rows) - - # Compute summary for last 7 days and last 2 days - runs_7day = sum(r['num_runs'] for r in test_rows) - failures_7day = sum(r['num_failures'] for r in test_rows) - runs_2day = sum(r['num_runs'] for r in test_rows if r['days_ago'] < 2) - failures_2day = sum(r['num_failures'] for r in test_rows if r['days_ago'] < 2) - - # Compute a sparkline (percentage failure for each day) - sparkline = [0 for x in xrange(8)] - for r in test_rows: - if r['num_runs'] > 0: - percent = float(r['num_failures']) / r['num_runs'] * 100 - else: - percent = 0 - sparkline[7 - r['days_ago']] = percent - - # Add to results list for tablet. - results.append(dict(test_name=test_name, - runs_7day=runs_7day, - failures_7day=failures_7day, - runs_2day=runs_2day, - failures_2day=failures_2day, - sparkline=",".join("%.2f" % p for p in sparkline))) - - return Template(""" -

Flaky rate over last week

- - - - - - - - {% for r in results %} - - - - - - - {% endfor %} -
testfailure rate (7-day)failure rate (2-day)trend
- {{ r.test_name |e }} - {{ r.failures_7day |e }} / {{ r.runs_7day }} - ({{ "%.2f"|format(r.failures_7day / r.runs_7day * 100) }}%) - {{ r.failures_2day |e }} / {{ r.runs_2day }} - {% if r.runs_2day > 0 %} - ({{ "%.2f"|format(r.failures_2day / r.runs_2day * 100) }}%) - {% endif %} - {{ r.sparkline |e }}
- - """).render(results=results) - - @cherrypy.expose - def list_failed_tests(self, build_pattern, num_days): - num_days = int(num_days) - c = self.execute_query( - """SELECT DISTINCT - test_name - FROM test_results - WHERE timestamp > NOW() - INTERVAL %(num_days)s DAY - AND status != 0 - AND build_id LIKE %(build_pattern)s""", - dict(build_pattern=build_pattern, - num_days=num_days)) - cherrypy.response.headers['Content-Type'] = 'text/plain' - return "\n".join(row['test_name'] for row in c.fetchall()) - - @cherrypy.expose - def test_drilldown(self, test_name): - - # Get summary statistics for the test, grouped by revision - c = self.execute_query( - """SELECT - revision, - MIN(timestamp) AS first_run, - SUM(IF(status != 0, 1, 0)) AS num_failures, - COUNT(*) AS num_runs - FROM test_results - WHERE timestamp > NOW() - INTERVAL 1 WEEK - AND test_name = %(test_name)s - GROUP BY revision - ORDER BY first_run DESC""", - dict(test_name=test_name)) - revision_rows = c.fetchall() - - # Convert to a dictionary, by revision - rev_dict = dict( [(r['revision'], r) for r in revision_rows] ) - - # Add an empty 'runs' array to each revision to be filled in below - for r in revision_rows: - r['runs'] = [] - - # Append the specific info on failures - c.execute("SELECT * from test_results " - "WHERE timestamp > NOW() - INTERVAL 1 WEEK " - "AND test_name = %(test_name)s " - "AND status != 0", - dict(test_name=test_name)) - for failure in c.fetchall(): - rev_dict[failure['revision']]['runs'].append(failure) - - return self.render_container(Template(""" -

{{ test_name |e }} flakiness over recent revisions

- {% for r in revision_rows %} -

{{ r.revision }} (Failed {{ r.num_failures }} / {{ r.num_runs }})

- {% if r.num_failures > 0 %} - - - - - - - - - {% for run in r.runs %} - - - - - - - - {% endfor %} -
timeconfigexit codemachinebuild
{{ run.timestamp |e }}{{ run.build_config |e }}{{ run.status |e }} - {% if run.log_key %} - failure log | - diagnose - {% endif %} - {{ run.hostname |e }}{{ run.build_id |e }}
- {% endif %} - {% endfor %} - """).render(revision_rows=revision_rows, test_name=test_name)) - - @cherrypy.expose - def index(self): - body = self.flaky_report_html() - body += "
" - body += self.recently_failed_html() - return self.render_container(body) - - def render_container(self, body): - """ Render the "body" HTML inside of a bootstrap container page. """ - template = Template(""" - - - Kudu test results - - - - - - - -
- {{ body }} -
- - - """) - return template.render(body=body) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - cherrypy.config.update( - {'server.socket_host': '0.0.0.0'} ) - cherrypy.quickstart(TRServer()) diff --git a/build-support/trigger_gerrit.py b/build-support/trigger_gerrit.py deleted file mode 100755 index 2788a53c5be5..000000000000 --- a/build-support/trigger_gerrit.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - - -# This tool triggers a Jenkins build based on a particular gerrit URL. -# The Jenkins build will post a +1 or -1 on the gerrit. -# -# NOTE: currently this is hard-coded to an internal Cloudera server. -# We plan to move to upstream infrastructure at some later date. - -import logging -import json -import re -import subprocess -import sys -import urllib -import urllib2 -import urlparse - -from yb_util import check_output - -GERRIT_HOST = "gerrit.cloudera.org" -JENKINS_URL = "http://sandbox.jenkins.cloudera.com/" -JENKINS_JOB = "kudu-public-gerrit" - - -def get_gerrit_ssh_command(): - url = check_output("git config --get remote.gerrit.url".split(" ")) - m = re.match(r'ssh://(.+)@(.+):(\d+)/.+', url) - if not m: - raise Exception("expected gerrit remote to be an ssh://user@host:port/ URL: %s" % url) - user, host, port = m.groups() - if host != GERRIT_HOST: - raise Exception("unexpected gerrit host %s in remote 'gerrit'. Expected %s" % ( - host, GERRIT_HOST)) - return ["ssh", "-p", port, "-l", user, host] - - -def current_ref_for_gerrit_number(change_num): - j = check_output(get_gerrit_ssh_command() + [ - "gerrit", "query", "--current-patch-set", "--format", "JSON", - "change:%d" % change_num]) - j = json.loads(j.split("\n")[0]) - return j['currentPatchSet']['ref'] - - -def url_to_ref(url): - u = urlparse.urlparse(url) - if not u.netloc.startswith(GERRIT_HOST): - print >>sys.stderr, "unexpected gerrit host %s, expected %s\n" % ( - u.netloc, GERRIT_HOST) - usage() - sys.exit(1) - if u.path == '/': - m = re.match(r'/c/(\d+)/', u.fragment) - if m: - return current_ref_for_gerrit_number(int(m.group(1))) - print >>sys.stderr, "invalid gerrit URL: ", url - usage() - sys.exit(1) - -def usage(): - print >>sys.stderr, "usage: %s \n" % sys.argv[0] - print >>sys.stderr, "The provided URL should look something like:" - print >>sys.stderr, "http://gerrit.cloudera.org:8080/#/c/963/\n" - - -def determine_ref(): - if len(sys.argv) != 2: - usage() - sys.exit(1) - - arg = sys.argv[1] - if arg.startswith("http"): - return url_to_ref(arg) - else: - print >>sys.stderr, "Unable to parse argument: %s\n" % arg - sys.exit(1) - - -def trigger_jenkins(ref): - logging.info("Will trigger Jenkins for ref %s" % ref) - url = "%s/job/%s/buildWithParameters" % (JENKINS_URL, JENKINS_JOB) - params = dict(GERRIT_BRANCH=ref) - req = urllib2.Request(url, - data=urllib.urlencode(params), - headers={"Accept": "application/json"}) - urllib2.urlopen(req).read() - logging.info("Successfuly triggered jenkins job!") - - -def main(): - gerrit_ref = determine_ref() - trigger_jenkins(gerrit_ref) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/python/MANIFEST.in b/python/MANIFEST.in deleted file mode 100644 index 597450cd7c19..000000000000 --- a/python/MANIFEST.in +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -include MANIFEST.in -include ../LICENSE.txt -include README.md -include setup.py - -graft kudu - -global-exclude *.so -global-exclude *.pyc -global-exclude *~ -global-exclude \#* -global-exclude .git* -global-exclude .DS_Store diff --git a/python/Makefile b/python/Makefile deleted file mode 100644 index e144a51075d6..000000000000 --- a/python/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -develop: - python setup.py build_ext --inplace - -all: develop - -clean-pyc: - find . -name "*.pyc" -exec rm -rf {} \; diff --git a/python/README.md b/python/README.md deleted file mode 100644 index 9a248bc1d0e9..000000000000 --- a/python/README.md +++ /dev/null @@ -1,12 +0,0 @@ -## kudu-python: Python interface to the Apache Kudu (incubating) C++ Client API - -Using this package requires that you install the Kudu C++ client libraries and -headers. See http://getkudu.io for more. - -To install from PyPI, run - -``` -pip install kudu-python -``` - -Installation from source requires Cython. diff --git a/python/kudu/__init__.pxd b/python/kudu/__init__.pxd deleted file mode 100644 index 217e5db96078..000000000000 --- a/python/kudu/__init__.pxd +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py deleted file mode 100644 index 0889c4a836f1..000000000000 --- a/python/kudu/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from kudu.client import (Client, Table, Scanner, Session, # noqa - Insert, Update, Delete, Predicate, - TimeDelta, YBError, - FLUSH_AUTO_BACKGROUND, - FLUSH_AUTO_SYNC, - FLUSH_MANUAL) - -from kudu.errors import (KuduException, KuduBadStatus, KuduNotFound, # noqa - KuduNotSupported, - KuduInvalidArgument) - -from kudu.schema import (int8, int16, int32, int64, string_ as string, # noqa - double_ as double, float_, binary, - timestamp, - KuduType, - SchemaBuilder, ColumnSpec, Schema, ColumnSchema, - COMPRESSION_DEFAULT, - COMPRESSION_NONE, - COMPRESSION_SNAPPY, - COMPRESSION_LZ4, - COMPRESSION_ZLIB, - ENCODING_AUTO, - ENCODING_PLAIN, - ENCODING_PREFIX, - ENCODING_GROUP_VARINT, - ENCODING_RLE) - - -def connect(host, port, admin_timeout_ms=None, rpc_timeout_ms=None): - """ - Connect to a Kudu master server - - Parameters - ---------- - host : string - Server address of master - port : int - Server port - admin_timeout_ms : int, optional - Admin timeout in milliseconds - rpc_timeout_ms : int, optional - RPC timeout in milliseconds - - Returns - ------- - client : kudu.Client - """ - addr = '{0}:{1}'.format(host, port) - return Client(addr, admin_timeout_ms=admin_timeout_ms, - rpc_timeout_ms=rpc_timeout_ms) - - -def timedelta(seconds=0, millis=0, micros=0, nanos=0): - """ - Construct a Kudu TimeDelta to set timeouts, etc. Use this function instead - of interacting with the TimeDelta class yourself. - - Returns - ------- - delta : kudu.client.TimeDelta - """ - from kudu.compat import long - # TimeDelta is a wrapper for kudu::MonoDelta - total_ns = (long(0) + seconds * long(1000000000) + - millis * long(1000000) + micros * long(1000) + nanos) - return TimeDelta.from_nanos(total_ns) - - -def schema_builder(): - """ - Create a kudu.SchemaBuilder instance - - Examples - -------- - builder = kudu.schema_builder() - builder.add_column('key1', kudu.int64, nullable=False) - builder.add_column('key2', kudu.int32, nullable=False) - - (builder.add_column('name', kudu.string) - .nullable() - .compression('lz4')) - - builder.add_column('value1', kudu.double) - builder.add_column('value2', kudu.int8, encoding='rle') - builder.set_primary_keys(['key1', 'key2']) - - schema = builder.build() - - Returns - ------- - builder : SchemaBuilder - """ - return SchemaBuilder() - - -from .version import version as __version__ # noqa diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx deleted file mode 100644 index 1ce6a6f4f5d1..000000000000 --- a/python/kudu/client.pyx +++ /dev/null @@ -1,1252 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -# distutils: language = c++ -# cython: embedsignature = True - -from libcpp.string cimport string -from libcpp cimport bool as c_bool - -cimport cpython -from cython.operator cimport dereference as deref - -from libyb_client cimport * - -from kudu.compat import tobytes, frombytes -from kudu.schema cimport Schema, ColumnSchema -from kudu.errors cimport check_status -from errors import KuduException - -import six - - -cdef class TimeDelta: - """ - Wrapper interface for kudu MonoDelta class, which is used to specify - timedeltas for timeouts and other uses. - """ - - cdef: - MonoDelta delta - - def __cinit__(self): - pass - - @staticmethod - def from_seconds(seconds): - """ - Construct a new TimeDelta from fractional seconds. - - Parameters - ---------- - seconds : double - - Returns - ------- - delta : TimeDelta - """ - cdef TimeDelta result = TimeDelta() - result.init(MonoDelta.FromSeconds(seconds)) - return result - - @staticmethod - def from_millis(int64_t ms): - """ - Construct a new TimeDelta from integer milliseconds. - - Parameters - ---------- - ms : int - - Returns - ------- - delta : TimeDelta - """ - cdef TimeDelta result = TimeDelta() - result.init(MonoDelta.FromMilliseconds(ms)) - return result - - @staticmethod - def from_micros(int64_t us): - """ - Construct a new TimeDelta from integer microseconds. - - Parameters - ---------- - us : int - - Returns - ------- - delta : TimeDelta - """ - cdef TimeDelta result = TimeDelta() - result.init(MonoDelta.FromMicroseconds(us)) - return result - - @staticmethod - def from_nanos(seconds): - """ - Construct a new TimeDelta from integer nanoseconds. - - Parameters - ---------- - ns : int - - Returns - ------- - delta : TimeDelta - """ - cdef TimeDelta result = TimeDelta() - result.init(MonoDelta.FromNanoseconds(seconds)) - return result - - cpdef double to_seconds(self): - """ - Return timedelta as fractional seconds. - """ - return self.delta.ToSeconds() - - cpdef int64_t to_millis(self): - """ - Return timedelta as exact milliseconds. - """ - return self.delta.ToMilliseconds() - - cpdef int64_t to_micros(self): - """ - Return timedelta as exact microseconds. - """ - return self.delta.ToMicroseconds() - - cpdef int64_t to_nanos(self): - """ - Return timedelta as exact nanoseconds. - """ - return self.delta.ToNanoseconds() - - cdef init(self, const MonoDelta& val): - self.delta = val - - def __repr__(self): - cdef object as_string - - if self.delta.Initialized(): - as_string = self.delta.ToString() - return 'kudu.TimeDelta({0})'.format(as_string) - else: - return 'kudu.TimeDelta()' - - def __richcmp__(TimeDelta self, TimeDelta other, int op): - if op == cpython.Py_EQ: - return self.delta.Equals(other.delta) - elif op == cpython.Py_NE: - return not self.delta.Equals(other.delta) - elif op == cpython.Py_LT: - return self.delta.LessThan(other.delta) - elif op == cpython.Py_LE: - return not self.delta.MoreThan(other.delta) - elif op == cpython.Py_GT: - return self.delta.MoreThan(other.delta) - elif op == cpython.Py_GE: - return not self.delta.LessThan(other.delta) - else: - raise ValueError('invalid operation: {0}'.format(op)) - - -cdef class Client: - - """ - The primary class for interacting with a Kudu cluster. Can connect to one - or more Kudu master servers. Do not instantiate this class directly; use - kudu.connect instead. - """ - - cdef: - shared_ptr[YBClient] client - YBClient* cp - - cdef readonly: - list master_addrs - - def __cinit__(self, addr_or_addrs, admin_timeout_ms=None, - rpc_timeout_ms=None): - cdef: - string c_addr - vector[string] c_addrs - YBClientBuilder builder - TimeDelta timeout - - if isinstance(addr_or_addrs, six.string_types): - addr_or_addrs = [addr_or_addrs] - elif not isinstance(addr_or_addrs, list): - addr_or_addrs = list(addr_or_addrs) - - self.master_addrs = addr_or_addrs - for addr in addr_or_addrs: - c_addrs.push_back(tobytes(addr)) - - builder.master_server_addrs(c_addrs) - - if admin_timeout_ms is not None: - timeout = TimeDelta.from_millis(admin_timeout_ms) - builder.default_admin_operation_timeout(timeout.delta) - - if rpc_timeout_ms is not None: - timeout = TimeDelta.from_millis(rpc_timeout_ms) - builder.default_rpc_timeout(timeout.delta) - - check_status(builder.Build(&self.client)) - - # A convenience - self.cp = self.client.get() - - def __dealloc__(self): - self.close() - - property is_multimaster: - - def __get__(self): - return self.cp.IsMultiMaster() - - cpdef close(self): - # Nothing yet to clean up here - pass - - def create_table(self, table_name, Schema schema): - """ - Creates a new Kudu table from the passed Schema and options. - - Parameters - ---------- - table_name : string - schema : kudu.Schema - Create using kudu.schema_builder - """ - cdef: - YBTableCreator* c - Status s - c = self.cp.NewTableCreator() - try: - s = (c.table_name(tobytes(table_name)) - .schema(schema.schema) - .Create()) - check_status(s) - finally: - del c - - def delete_table(self, table_name): - """ - Delete a Kudu table. Raises KuduNotFound if the table does not exist. - - Parameters - ---------- - table_name : string - """ - check_status(self.cp.DeleteTable(tobytes(table_name))) - - def drop_if_exists(self, table_name): - """Return True if the indicated table exists in the Kudu cluster. - - Parameters - ---------- - table_name : string - - Returns - ------- - exists : bool - - """ - cdef: - string c_name = tobytes(table_name) - c_bool exists - - check_status(self.cp.TableExists(c_name, &exists)) - return exists - - def table(self, table_name): - """ - Construct a kudu.Table and retrieve its schema from the cluster. - - Raises KuduNotFound if the table does not exist. - - Parameters - ---------- - table_name : string - - Returns - ------- - table : kudu.Table - """ - table_name = tobytes(table_name) - cdef Table table = Table(table_name, self) - - check_status(self.cp.OpenTable(table_name, &table.table)) - table.init() - return table - - def list_tables(self, match_substring=None): - """ - Retrieve a list of table names in the Kudu cluster with an optional - substring filter. - - Parameters - ---------- - match_substring : string, optional - If passed, the string must be exactly contained in the table names - - Returns - ------- - tables : list[string] - Table names returned from Kudu - """ - cdef: - vector[string] tables - string c_match - size_t i - - if match_substring is not None: - c_match = tobytes(match_substring) - check_status(self.cp.ListTables(&tables, c_match)) - else: - check_status(self.cp.ListTables(&tables)) - - result = [] - for i in range(tables.size()): - result.append(frombytes(tables[i])) - return result - - def new_session(self, flush_mode='manual', timeout_ms=5000): - """ - Create a new YBSession for applying write operations. - - Parameters - ---------- - flush_mode : {'manual', 'sync', 'background'}, default 'manual' - See Session.set_flush_mode - timeout_ms : int, default 5000 - Timeout in milliseconds - - Returns - ------- - session : kudu.Session - """ - cdef Session result = Session() - result.s = self.cp.NewSession() - - result.set_flush_mode(flush_mode) - result.set_timeout_ms(timeout_ms) - - return result - - - -#---------------------------------------------------------------------- -# Handle marshalling Python values to raw values. Since range predicates -# require a const void*, this is one valid (though a bit verbose) -# approach. Note that later versions of Cython handle many Python -> C type -# casting problems (and integer overflows), but these should all be tested -# rigorously in our test suite - - -cdef class RawValue: - cdef: - void* data - - def __cinit__(self): - self.data = NULL - - -cdef class Int8Val(RawValue): - cdef: - int8_t val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class Int16Val(RawValue): - cdef: - int16_t val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class Int32Val(RawValue): - cdef: - int32_t val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class Int64Val(RawValue): - cdef: - int64_t val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class DoubleVal(RawValue): - cdef: - double val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class FloatVal(RawValue): - cdef: - float val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class BoolVal(RawValue): - cdef: - c_bool val - - def __cinit__(self, obj): - self.val = obj - self.data = &self.val - - -cdef class StringVal(RawValue): - cdef: - # Python "str" object that was passed into the constructor. - # We hold a reference to this so that the underlying data - # doesn't go out of scope. - object py_str - # Heap-allocated Slice object, owned by this instance, - # which points to the data in 'py_str' - cdef Slice* val - - def __cinit__(self, obj): - self.py_str = obj - self.val = new Slice(self.py_str, len(self.py_str)) - # The C++ API expects a Slice* to be passed to the range predicate - # constructor. - self.data = self.val - - def __dealloc__(self): - del self.val - -#---------------------------------------------------------------------- - - -cdef class Table: - - """ - Represents a Kudu table, containing the schema and other tools. Create by - using the kudu.Client.table method after connecting to a cluster. - """ - - cdef: - shared_ptr[YBTable] table - - cdef readonly: - object _name - Schema schema - Client parent - - def __cinit__(self, name, Client client): - self._name = name - self.parent = client - - # Users should not instantiate directly - self.schema = Schema() - - cdef init(self): - # Called after the refptr has been populated - self.schema.schema = &self.ptr().schema() - self.schema.own_schema = 0 - self.schema.parent = self - - def __len__(self): - # TODO: is this cheaply knowable? - raise NotImplementedError - - def __getitem__(self, key): - spec = self.schema[key] - return Column(self, key, spec) - - property name: - - def __get__(self): - return frombytes(self.ptr().name()) - - # XXX: don't love this name - property num_columns: - - def __get__(self): - return len(self.schema) - - def rename(self, new_name): - raise NotImplementedError - - def drop(self): - raise NotImplementedError - - def new_insert(self): - """ - Create a new Insert operation. Pass the completed Insert to a Session. - - Returns - ------- - insert : Insert - """ - return Insert(self) - - def new_update(self): - """ - Create a new Update operation. Pass the completed Update to a Session. - - Returns - ------- - update : Update - """ - return Update(self) - - def new_delete(self): - """ - Create a new Delete operation. Pass the completed Update to a Session. - - Returns - ------- - delete : Delete - """ - return Delete(self) - - def scanner(self): - """ - Create a new scanner for this table for retrieving a selection of table - rows. - - Examples - -------- - scanner = table.scanner() - scanner.add_predicate(table['key'] > 10) - scanner.open() - batch = scanner.read_all() - tuples = batch.as_tuples() - - Returns - ------- - scanner : kudu.Scanner - """ - cdef Scanner result = Scanner(self) - result.scanner = new YBScanner(self.ptr()) - return result - - cdef inline YBTable* ptr(self): - return self.table.get() - - -cdef class Column: - - """ - A reference to a Kudu table column intended to simplify creating predicates - and other column-specific operations. - - Write arithmetic comparisons to create new Predicate objects that can be - passed to a Scanner. - - Examples - -------- - scanner.add_predicate(table[col_name] <= 10) - """ - cdef readonly: - object name - Table parent - ColumnSchema spec - - def __cinit__(self, Table parent, object name, ColumnSchema spec): - self.name = tobytes(name) - self.parent = parent - self.spec = spec - - def __repr__(self): - result = ('Column({0}, parent={1}, type={2})' - .format(frombytes(self.name), - self.parent.name, - self.spec.type.name)) - return result - - cdef YBValue* box_value(self, object obj) except NULL: - cdef: - YBValue* val - Slice* slc - - if isinstance(obj, unicode): - obj = obj.encode('utf8') - - if isinstance(obj, bytes): - slc = new Slice( obj, len(obj)) - val = YBValue.CopyString(deref(slc)) - del slc - elif isinstance(obj, int): - val = YBValue.FromInt(obj) - elif isinstance(obj, float): - val = YBValue.FromDouble(obj) - else: - raise TypeError(obj) - - return val - - def __richcmp__(Column self, value, int op): - cdef: - YBPredicate* pred - YBValue* val - Slice* col_name_slice - ComparisonOp cmp_op - Predicate result - - col_name_slice = new Slice( self.name, - len(self.name)) - - try: - if op == 1: # <= - cmp_op = KUDU_LESS_EQUAL - elif op == 2: # == - cmp_op = KUDU_EQUAL - elif op == 5: # >= - cmp_op = KUDU_GREATER_EQUAL - else: - raise NotImplementedError - - val = self.box_value(value) - pred = (self.parent.ptr() - .NewComparisonPredicate(deref(col_name_slice), - cmp_op, val)) - finally: - del col_name_slice - - result = Predicate() - result.init(pred) - - return result - - -cdef class Predicate: - - """ - Wrapper for a YBPredicate. Pass to Scanner.add_predicates - """ - - cdef: - YBPredicate* pred - - def __cinit__(self): - self.pred = NULL - - def __dealloc__(self): - if self.pred != NULL: - del self.pred - - cdef init(self, YBPredicate* pred): - self.pred = pred - - -FLUSH_AUTO_SYNC = FlushMode_AutoSync -FLUSH_AUTO_BACKGROUND = FlushMode_AutoBackground -FLUSH_MANUAL = FlushMode_Manual - -cdef dict _flush_modes = { - 'manual': FlushMode_Manual, - 'sync': FlushMode_AutoSync, - 'background': FlushMode_AutoBackground -} - - -cdef class Session: - """ - Wrapper for a client YBSession to build up write operations to interact - with the cluster. - """ - cdef: - shared_ptr[YBSession] s - - def __cinit__(self): - pass - - def set_flush_mode(self, flush_mode='manual'): - """ - Set the session operation flush mode - - Parameters - ---------- - flush_mode : {'manual', 'sync', 'background'}, default 'manual' - You can also use the constants FLUSH_MANUAL, FLUSH_AUTO_SYNC, - and FLUSH_AUTO_BACKGROUND - """ - cdef Status status - cdef FlushMode fmode - - if isinstance(flush_mode, int): - # todo: validation - fmode = flush_mode - else: - try: - fmode = _flush_modes[flush_mode.lower()] - except KeyError: - raise ValueError('Invalid flush mode: {0}' - .format(flush_mode)) - - status = self.s.get().SetFlushMode(fmode) - - check_status(status) - - def set_timeout_ms(self, int64_t ms): - """ - Set the session timeout in milliseconds - """ - self.s.get().SetTimeoutMillis(ms) - - def apply(self, WriteOperation op): - """ - Apply the indicated write operation - - Examples - -------- - # Executes a single Insert operation - session = client.new_session() - op = table.new_insert() - op['key'] = 0 - op['value1'] = 5 - op['value2'] = 3.5 - session.apply(op) - session.flush() - """ - return op.add_to_session(self) - - def flush(self): - """ - Flush pending operations - """ - check_status(self.s.get().Flush()) - - def get_pending_errors(self): - """ - Returns a list of buffered Kudu errors. A second value is returned - indicating if there were more errors than could be stored in the - session's error buffer (i.e. False means there was no error overflow) - - Returns - ------- - errors, overflowed : list, bool - """ - cdef: - YBError error - vector[C_YBError*] v_errors - c_bool overflowed - size_t i - - self.s.get().GetPendingErrors(&v_errors, &overflowed) - - result = [] - for i in range(v_errors.size()): - error = YBError() - error.error = v_errors[i] - result.append(error) - - return result, overflowed - - -cdef class Row: - - """ - A single row from a row batch - """ - - cdef: - # So we can access the schema information - Table table - - RowBatch parent - - # This object is owned by the parent RowBatch - YBRowResult* row - - def __cinit__(self, batch, table): - self.parent = batch - self.table = table - self.row = NULL - - def __dealloc__(self): - pass - - cdef tuple as_tuple(self): - """ - Return the row as a Python tuple - """ - cdef: - int i, k - tuple tup - - k = self.table.num_columns - tup = cpython.PyTuple_New(k) - for i in range(k): - val = None - - if not self.is_null(i): - val = self.get_slot(i) - - cpython.Py_INCREF(val) - cpython.PyTuple_SET_ITEM(tup, i, val) - - return tup - - cdef inline get_bool(self, int i): - cdef c_bool val - check_status(self.row.GetBool(i, &val)) - # The built-in bool is masked by the libcpp typedef - return bool(val) - - cdef inline get_int8(self, int i): - cdef int8_t val - check_status(self.row.GetInt8(i, &val)) - return val - - cdef inline get_int16(self, int i): - cdef int16_t val - check_status(self.row.GetInt16(i, &val)) - return val - - cdef inline get_int32(self, int i): - cdef int32_t val - check_status(self.row.GetInt32(i, &val)) - return val - - cdef inline get_int64(self, int i): - cdef int64_t val - check_status(self.row.GetInt64(i, &val)) - return val - - cdef inline get_double(self, int i): - cdef double val - check_status(self.row.GetDouble(i, &val)) - return val - - cdef inline get_float(self, int i): - cdef float val - check_status(self.row.GetFloat(i, &val)) - return val - - cdef inline get_string(self, int i): - cdef Slice val - check_status(self.row.GetString(i, &val)) - return cpython.PyBytes_FromStringAndSize( val.mutable_data(), - val.size()) - - cdef inline get_slot(self, int i): - cdef: - Status s - DataType t = self.table.schema.loc_type(i) - - if t == YB_BOOL: - return self.get_bool(i) - elif t == YB_INT8: - return self.get_int8(i) - elif t == YB_INT16: - return self.get_int16(i) - elif t == YB_INT32: - return self.get_int32(i) - elif t == YB_INT64: - return self.get_int64(i) - elif t == YB_DOUBLE: - return self.get_double(i) - elif t == YB_FLOAT: - return self.get_float(i) - elif t == YB_STRING: - return frombytes(self.get_string(i)) - else: - raise TypeError(t) - - cdef inline bint is_null(self, int i): - return self.row.IsNull(i) - - -cdef class RowBatch: - """ - Class holding a batch of rows from a Scanner - """ - # This class owns the YBRowResult data - cdef: - Table table - vector[YBRowResult] rows - - def __cinit__(self, Table table): - self.table = table - - def __len__(self): - return self.rows.size() - - def __getitem__(self, i): - return self.get_row(i).as_tuple() - - def __iter__(self): - cdef int i = 0 - for i in range(len(self)): - yield self.get_row(i).as_tuple() - - def as_tuples(self): - """ - Return RowBatch as a list of Python tuples - - To simplify testing for the moment. - """ - cdef list tuples = [] - for i in range(self.rows.size()): - tuples.append(self.get_row(i).as_tuple()) - return tuples - - cdef Row get_row(self, i): - # TODO: boundscheck - - # For safety, we need to increment the parent reference count and hold - # on to a reference internally so that if the RowBatch goes out of - # scope we won't end up with orphaned Row objects. This isn't the best, - # but an intermediate solution until we can do something better.. - # - # One alternative is to copy the YBRowResult into the Row, but that - # doesn't feel right. - cdef Row row = Row(self, self.table) - row.row = &self.rows[i] - - return row - - -cdef class Scanner: - """ - A class for defining a selection of data we wish to scan out of a Kudu - table. Create a scanner using Table.scanner. - """ - - cdef: - Table table - YBScanner* scanner - bint is_open - - def __cinit__(self, Table table): - self.table = table - self.scanner = NULL - self.is_open = 0 - - def __dealloc__(self): - # We own this one - if self.scanner != NULL: - del self.scanner - - cdef inline ensure_open(self): - if not self.is_open: - self.open() - - def add_predicates(self, preds): - """ - Add a list of scan predicates to the scanner. Select columns from the - parent table and make comparisons to create predicates. - - Examples - -------- - c = table[col_name] - preds = [c >= 0, c <= 10] - scanner.add_predicates(preds) - - Parameters - ---------- - preds : list of Predicate - """ - for pred in preds: - self.add_predicate(pred) - - cpdef add_predicate(self, Predicate pred): - """ - Add a scan predicates to the scanner. Select columns from the - parent table and make comparisons to create predicates. - - Examples - -------- - pred = table[col_name] <= 10 - scanner.add_predicate(pred) - - Parameters - ---------- - pred : kudu.Predicate - """ - cdef YBPredicate* clone - - # We clone the YBPredicate so that the Predicate wrapper class can be - # reused - clone = pred.pred.Clone() - check_status(self.scanner.AddConjunctPredicate(clone)) - - def set_fault_tolerant(self): - """ - Makes the underlying YBScanner fault tolerant. - Returns a reference to itself to facilitate chaining. - - Returns - ------- - self : Scanner - """ - check_status(self.scanner.SetFaultTolerant()) - return self - - def open(self): - """ - Returns a reference to itself to facilitate chaining - - Returns - ------- - self : Scanner - """ - if not self.is_open: - check_status(self.scanner.Open()) - self.is_open = 1 - return self - - def has_more_rows(self): - """ - Returns True if there are more rows to be read. - """ - return self.scanner.HasMoreRows() - - def read_all_tuples(self): - """ - Compute a RowBatch containing all rows from the scan operation (which - hopefully fit into memory, probably not handled gracefully at the - moment). - """ - cdef list tuples = [] - cdef RowBatch batch - - self.ensure_open() - - while self.has_more_rows(): - batch = self.next_batch() - tuples.extend(batch.as_tuples()) - - return tuples - - def read_next_batch_tuples(self): - return self.next_batch().as_tuples() - - cdef RowBatch next_batch(self): - """ - Retrieve the next batch of rows from the scanner. - - Returns - ------- - batch : RowBatch - """ - if not self.has_more_rows(): - raise StopIteration - - cdef RowBatch batch = RowBatch(self.table) - check_status(self.scanner.NextBatch(&batch.rows)) - return batch - - -cdef class YBError: - - """ - Wrapper for a C++ YBError indicating a client error resulting from - applying operations in a session. - """ - - cdef: - C_YBError* error - - def __cinit__(self): - self.error = NULL - - def __dealloc__(self): - # We own this object - if self.error != NULL: - del self.error - - def failed_op(self): - raise NotImplementedError - - def __repr__(self): - return "YBError('%s')" % (self.error.status().ToString()) - - -cdef class WriteOperation: - cdef: - Table table - YBPartialRow* row - bint applied - - def __cinit__(self, Table table): - # This gets called before any subclass cinit methods - self.table = table - self.applied = 0 - - def __setitem__(self, key, value): - if isinstance(key, basestring): - self.set_field(key, value) - else: - self.set_loc(key, value) - - cpdef set_field(self, key, value): - cdef: - int i = self.table.schema.get_loc(key) - DataType t = self.table.schema.loc_type(i) - cdef Slice* slc - - # Leave it to Cython to do the coercion and complain if it doesn't - # work. Cython will catch many casting problems but we should verify - # with unit tests. - if t == YB_BOOL: - self.row.SetBool(i, value) - elif t == YB_INT8: - self.row.SetInt8(i, value) - elif t == YB_INT16: - self.row.SetInt16(i, value) - elif t == YB_INT32: - self.row.SetInt32(i, value) - elif t == YB_INT64: - self.row.SetInt64(i, value) - elif t == YB_FLOAT: - self.row.SetFloat(i, value) - elif t == YB_DOUBLE: - self.row.SetDouble(i, value) - elif t == YB_STRING: - if not cpython.PyBytes_Check(value): - value = value.encode('utf8') - - # TODO: It would be much better not to heap-allocate a Slice object - slc = new Slice(cpython.PyBytes_AsString(value)) - - # Not safe to take a reference to PyBytes data for now - self.row.SetStringCopy(i, deref(slc)) - del slc - - cpdef set_loc(self, int i, value): - pass - - cpdef set_field_null(self, key): - pass - - cpdef set_loc_null(self, int i): - pass - - cdef add_to_session(self, Session s): - pass - - -cdef class Insert(WriteOperation): - cdef: - YBInsert* op - - def __cinit__(self, Table table): - self.op = self.table.ptr().NewInsert() - self.row = self.op.mutable_row() - - def __dealloc__(self): - del self.op - - cdef add_to_session(self, Session s): - if self.applied: - raise Exception - - check_status(s.s.get().Apply(self.op)) - self.op = NULL - self.applied = 1 - - -cdef class Update(WriteOperation): - cdef: - YBUpdate* op - - def __cinit__(self, Table table): - self.table = table - self.op = table.ptr().NewUpdate() - self.row = self.op.mutable_row() - - def __dealloc__(self): - del self.op - - cdef add_to_session(self, Session s): - pass - - -cdef class Delete(WriteOperation): - cdef: - YBDelete* op - - def __cinit__(self, Table table): - self.table = table - self.op = table.ptr().NewDelete() - self.row = self.op.mutable_row() - - def __dealloc__(self): - del self.op - - cdef add_to_session(self, Session s): - if self.applied: - raise Exception - - check_status(s.s.get().Apply(self.op)) - self.applied = 1 - self.op = NULL - - - -cdef inline cast_pyvalue(DataType t, object o): - if t == YB_BOOL: - return BoolVal(o) - elif t == YB_INT8: - return Int8Val(o) - elif t == YB_INT16: - return Int16Val(o) - elif t == YB_INT32: - return Int32Val(o) - elif t == YB_INT64: - return Int64Val(o) - elif t == YB_DOUBLE: - return DoubleVal(o) - elif t == YB_FLOAT: - return FloatVal(o) - elif t == YB_STRING: - return StringVal(o) - else: - raise TypeError(t) diff --git a/python/kudu/compat.py b/python/kudu/compat.py deleted file mode 100644 index e4e7c2b788de..000000000000 --- a/python/kudu/compat.py +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -# flake8: noqa - -import itertools - -import numpy as np - -import sys -import six -from six import BytesIO, StringIO, string_types as py_string - - -PY26 = sys.version_info[:2] == (2, 6) -PY2 = sys.version_info[0] == 2 - - -if PY26: - import unittest2 as unittest -else: - import unittest - - -if PY2: - import cPickle - - try: - from cdecimal import Decimal - except ImportError: - from decimal import Decimal - - unicode_type = unicode - lzip = zip - zip = itertools.izip - - def dict_values(x): - return x.values() - - range = xrange - long = long - - def tobytes(o): - if isinstance(o, unicode): - return o.encode('utf8') - else: - return o - - def frombytes(o): - return o -else: - unicode_type = str - def lzip(*x): - return list(zip(*x)) - long = int - zip = zip - def dict_values(x): - return list(x.values()) - from decimal import Decimal - range = range - - def tobytes(o): - if isinstance(o, str): - return o.encode('utf8') - else: - return o - - def frombytes(o): - return o.decode('utf8') - - -integer_types = six.integer_types + (np.integer,) diff --git a/python/kudu/errors.pxd b/python/kudu/errors.pxd deleted file mode 100644 index 3bf2e765a3be..000000000000 --- a/python/kudu/errors.pxd +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from libyb_client cimport * - -cdef check_status(const Status& status) diff --git a/python/kudu/errors.pyx b/python/kudu/errors.pyx deleted file mode 100644 index 71f1e325a39c..000000000000 --- a/python/kudu/errors.pyx +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -class KuduException(Exception): - pass - - -class KuduBadStatus(KuduException): - """ - A Kudu C++ client operation returned an error Status - """ - pass - - -class KuduNotFound(KuduBadStatus): - pass - - -class KuduNotSupported(KuduBadStatus): - pass - - -class KuduInvalidArgument(KuduBadStatus): - pass - - -class KuduNotAuthorized(KuduBadStatus): - pass - - -class KuduAborted(KuduBadStatus): - pass - - -cdef check_status(const Status& status): - if status.ok(): - return - - cdef string c_message = status.message().ToString() - - if status.IsNotFound(): - raise KuduNotFound(c_message) - elif status.IsNotSupported(): - raise KuduNotSupported(c_message) - elif status.IsInvalidArgument(): - raise KuduInvalidArgument(c_message) - else: - raise KuduBadStatus(status.ToString()) diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd deleted file mode 100644 index 6a89816bc860..000000000000 --- a/python/kudu/libkudu_client.pxd +++ /dev/null @@ -1,607 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# distutils: language = c++ - -from libc.stdint cimport * -from libcpp cimport bool as c_bool -from libcpp.string cimport string -from libcpp.vector cimport vector - -# This must be included for cerr and other things to work -cdef extern from "": - pass - -#---------------------------------------------------------------------- -# Smart pointers and such - -cdef extern from "yb/client/shared_ptr.h" namespace "yb::client::sp" nogil: - - cdef cppclass shared_ptr[T]: - T* get() - void reset() - void reset(T* p) - -cdef extern from "yb/util/status.h" namespace "yb" nogil: - - # We can later add more of the common status factory methods as needed - cdef Status Status_OK "Status::OK"() - - cdef cppclass Status: - Status() - - string ToString() - - Slice message() - - c_bool ok() - c_bool IsNotFound() - c_bool IsCorruption() - c_bool IsNotSupported() - c_bool IsIOError() - c_bool IsInvalidArgument() - c_bool IsAlreadyPresent() - c_bool IsRuntimeError() - c_bool IsNetworkError() - c_bool IsIllegalState() - c_bool IsNotAuthorized() - c_bool IsAborted() - - -cdef extern from "yb/util/monotime.h" namespace "yb" nogil: - - # These classes are not yet needed directly but will need to be completed - # from the C++ API - cdef cppclass MonoDelta: - MonoDelta() - - @staticmethod - MonoDelta FromSeconds(double seconds) - - @staticmethod - MonoDelta FromMilliseconds(int64_t ms) - - @staticmethod - MonoDelta FromMicroseconds(int64_t us) - - @staticmethod - MonoDelta FromNanoseconds(int64_t ns) - - c_bool Initialized() - c_bool LessThan(const MonoDelta& other) - c_bool MoreThan(const MonoDelta& other) - c_bool Equals(const MonoDelta& other) - - string ToString() - - double ToSeconds() - int64_t ToMilliseconds() - int64_t ToMicroseconds() - int64_t ToNanoseconds() - - # TODO, when needed - # void ToTimeVal(struct timeval *tv) - # void ToTimeSpec(struct timespec *ts) - - # @staticmethod - # void NanosToTimeSpec(int64_t nanos, struct timespec* ts); - - - cdef cppclass MonoTime: - pass - - -cdef extern from "yb/client/schema.h" namespace "yb::client" nogil: - - enum DataType" yb::client::YBColumnSchema::DataType": - YB_INT8 " yb::client::YBColumnSchema::INT8" - YB_INT16 " yb::client::YBColumnSchema::INT16" - YB_INT32 " yb::client::YBColumnSchema::INT32" - YB_INT64 " yb::client::YBColumnSchema::INT64" - YB_STRING " yb::client::YBColumnSchema::STRING" - YB_BOOL " yb::client::YBColumnSchema::BOOL" - YB_FLOAT " yb::client::YBColumnSchema::FLOAT" - YB_DOUBLE " yb::client::YBColumnSchema::DOUBLE" - YB_BINARY " yb::client::YBColumnSchema::BINARY" - YB_TIMESTAMP " yb::client::YBColumnSchema::TIMESTAMP" - - enum EncodingType" yb::client::YBColumnStorageAttributes::EncodingType": - EncodingType_AUTO " yb::client::YBColumnStorageAttributes::AUTO_ENCODING" - EncodingType_PLAIN " yb::client::YBColumnStorageAttributes::PLAIN_ENCODING" - EncodingType_PREFIX " yb::client::YBColumnStorageAttributes::PREFIX_ENCODING" - EncodingType_GROUP_VARINT " yb::client::YBColumnStorageAttributes::GROUP_VARINT" - EncodingType_RLE " yb::client::YBColumnStorageAttributes::RLE" - - enum CompressionType" yb::client::YBColumnStorageAttributes::CompressionType": - CompressionType_DEFAULT " yb::client::YBColumnStorageAttributes::DEFAULT_COMPRESSION" - CompressionType_NONE " yb::client::YBColumnStorageAttributes::NO_COMPRESSION" - CompressionType_SNAPPY " yb::client::YBColumnStorageAttributes::SNAPPY" - CompressionType_LZ4 " yb::client::YBColumnStorageAttributes::LZ4" - CompressionType_ZLIB " yb::client::YBColumnStorageAttributes::ZLIB" - - cdef struct YBColumnStorageAttributes: - YBColumnStorageAttributes() - - EncodingType encoding - CompressionType compression - string ToString() - - cdef cppclass YBColumnSchema: - YBColumnSchema(const YBColumnSchema& other) - YBColumnSchema(const string& name, DataType type) - YBColumnSchema(const string& name, DataType type, c_bool is_nullable) - YBColumnSchema(const string& name, DataType type, c_bool is_nullable, - const void* default_value) - - string& name() - c_bool is_nullable() - DataType type() - - c_bool Equals(YBColumnSchema& other) - void CopyFrom(YBColumnSchema& other) - - cdef cppclass YBSchema: - YBSchema() - YBSchema(vector[YBColumnSchema]& columns, int key_columns) - - c_bool Equals(const YBSchema& other) - YBColumnSchema Column(size_t idx) - size_t num_columns() - - void GetPrimaryKeyColumnIndexes(vector[int]* indexes) - - YBPartialRow* NewRow() - - cdef cppclass YBColumnSpec: - - YBColumnSpec* Default(YBValue* value) - YBColumnSpec* RemoveDefault() - - YBColumnSpec* Compression(CompressionType compression) - YBColumnSpec* Encoding(EncodingType encoding) - YBColumnSpec* BlockSize(int32_t block_size) - - YBColumnSpec* PrimaryKey() - YBColumnSpec* NotNull() - YBColumnSpec* Nullable() - YBColumnSpec* Type(DataType type_) - - YBColumnSpec* RenameTo(string& new_name) - - - cdef cppclass YBSchemaBuilder: - - YBColumnSpec* AddColumn(string& name) - YBSchemaBuilder* SetPrimaryKey(vector[string]& key_col_names); - - Status Build(YBSchema* schema) - - -cdef extern from "yb/client/row_result.h" namespace "yb::client" nogil: - - cdef cppclass YBRowResult: - c_bool IsNull(Slice& col_name) - c_bool IsNull(int col_idx) - - # These getters return a bad Status if the type does not match, - # the value is unset, or the value is NULL. Otherwise they return - # the current set value in *val. - Status GetBool(Slice& col_name, c_bool* val) - - Status GetInt8(Slice& col_name, int8_t* val) - Status GetInt16(Slice& col_name, int16_t* val) - Status GetInt32(Slice& col_name, int32_t* val) - Status GetInt64(Slice& col_name, int64_t* val) - - Status GetTimestamp(const Slice& col_name, - int64_t* micros_since_utc_epoch) - - Status GetBool(int col_idx, c_bool* val) - - Status GetInt8(int col_idx, int8_t* val) - Status GetInt16(int col_idx, int16_t* val) - Status GetInt32(int col_idx, int32_t* val) - Status GetInt64(int col_idx, int64_t* val) - - Status GetString(Slice& col_name, Slice* val) - Status GetString(int col_idx, Slice* val) - - Status GetFloat(Slice& col_name, float* val) - Status GetFloat(int col_idx, float* val) - - Status GetDouble(Slice& col_name, double* val) - Status GetDouble(int col_idx, double* val) - - Status GetBinary(const Slice& col_name, Slice* val) - Status GetBinary(int col_idx, Slice* val) - - const void* cell(int col_idx) - string ToString() - - -cdef extern from "yb/util/slice.h" namespace "kudu" nogil: - - cdef cppclass Slice: - Slice() - Slice(const uint8_t* data, size_t n) - Slice(const char* data, size_t n) - - Slice(string& s) - Slice(const char* s) - - # Many other constructors have been omitted; we can return and add them - # as needed for the code generation. - - const uint8_t* data() - uint8_t* mutable_data() - size_t size() - c_bool empty() - - uint8_t operator[](size_t n) - - void clear() - void remove_prefix(size_t n) - void truncate(size_t n) - - Status check_size(size_t expected_size) - - string ToString() - - string ToDebugString() - string ToDebugString(size_t max_len) - - int compare(Slice& b) - - c_bool starts_with(Slice& x) - - void relocate(uint8_t* d) - - # Many other API methods omitted - - -cdef extern from "yb/common/partial_row.h" namespace "kudu" nogil: - - cdef cppclass YBPartialRow: - # Schema must not be garbage-collected - # YBPartialRow(const Schema* schema) - - #---------------------------------------------------------------------- - # Setters - - # Slice setters - Status SetBool(Slice& col_name, c_bool val) - - Status SetInt8(Slice& col_name, int8_t val) - Status SetInt16(Slice& col_name, int16_t val) - Status SetInt32(Slice& col_name, int32_t val) - Status SetInt64(Slice& col_name, int64_t val) - - Status SetTimestamp(const Slice& col_name, - int64_t micros_since_utc_epoch) - Status SetTimestamp(int col_idx, int64_t micros_since_utc_epoch) - - Status SetDouble(Slice& col_name, double val) - Status SetFloat(Slice& col_name, float val) - - # Integer setters - Status SetBool(int col_idx, c_bool val) - - Status SetInt8(int col_idx, int8_t val) - Status SetInt16(int col_idx, int16_t val) - Status SetInt32(int col_idx, int32_t val) - Status SetInt64(int col_idx, int64_t val) - - Status SetDouble(int col_idx, double val) - Status SetFloat(int col_idx, float val) - - # Set, but does not copy string - Status SetString(Slice& col_name, Slice& val) - Status SetString(int col_idx, Slice& val) - - Status SetStringCopy(Slice& col_name, Slice& val) - Status SetStringCopy(int col_idx, Slice& val) - - Status SetBinaryCopy(const Slice& col_name, const Slice& val) - Status SetBinaryCopy(int col_idx, const Slice& val) - - Status SetNull(Slice& col_name) - Status SetNull(int col_idx) - - Status Unset(Slice& col_name) - Status Unset(int col_idx) - - #---------------------------------------------------------------------- - # Getters - - c_bool IsColumnSet(Slice& col_name) - c_bool IsColumnSet(int col_idx) - - c_bool IsNull(Slice& col_name) - c_bool IsNull(int col_idx) - - Status GetBool(Slice& col_name, c_bool* val) - Status GetBool(int col_idx, c_bool* val) - - Status GetInt8(Slice& col_name, int8_t* val) - Status GetInt8(int col_idx, int8_t* val) - - Status GetInt16(Slice& col_name, int16_t* val) - Status GetInt16(int col_idx, int16_t* val) - - Status GetInt32(Slice& col_name, int32_t* val) - Status GetInt32(int col_idx, int32_t* val) - - Status GetInt64(Slice& col_name, int64_t* val) - Status GetInt64(int col_idx, int64_t* val) - - Status GetTimestamp(const Slice& col_name, - int64_t* micros_since_utc_epoch) - Status GetTimestamp(int col_idx, int64_t* micros_since_utc_epoch) - - Status GetDouble(Slice& col_name, double* val) - Status GetDouble(int col_idx, double* val) - - Status GetFloat(Slice& col_name, float* val) - Status GetFloat(int col_idx, float* val) - - # Gets the string but does not copy the value. Callers should - # copy the resulting Slice if necessary. - Status GetString(Slice& col_name, Slice* val) - Status GetString(int col_idx, Slice* val) - - Status GetBinary(const Slice& col_name, Slice* val) - Status GetBinary(int col_idx, Slice* val) - - Status EncodeRowKey(string* encoded_key) - string ToEncodedRowKeyOrDie() - - # Return true if all of the key columns have been specified - # for this mutation. - c_bool IsKeySet() - - # Return true if all columns have been specified. - c_bool AllColumnsSet() - string ToString() - - # const Schema* schema() - - -cdef extern from "yb/client/write_op.h" namespace "yb::client" nogil: - - enum WriteType" yb::client::YBWriteOperation::Type": - INSERT " yb::client::YBWriteOperation::INSERT" - UPDATE " yb::client::YBWriteOperation::UPDATE" - DELETE " yb::client::YBWriteOperation::DELETE" - - cdef cppclass YBWriteOperation: - YBPartialRow& row() - YBPartialRow* mutable_row() - - # This is a pure virtual function implemented on each of the cppclass - # subclasses - string ToString() - - # Also a pure virtual - WriteType type() - - cdef cppclass YBInsert(YBWriteOperation): - pass - - cdef cppclass YBDelete(YBWriteOperation): - pass - - cdef cppclass YBUpdate(YBWriteOperation): - pass - - -cdef extern from "yb/client/scan_predicate.h" namespace "yb::client" nogil: - enum ComparisonOp" yb::client::YBPredicate::ComparisonOp": - KUDU_LESS_EQUAL " yb::client::YBPredicate::LESS_EQUAL" - KUDU_GREATER_EQUAL " yb::client::YBPredicate::GREATER_EQUAL" - KUDU_EQUAL " yb::client::YBPredicate::EQUAL" - - cdef cppclass YBPredicate: - YBPredicate* Clone() - - -cdef extern from "yb/client/value.h" namespace "yb::client" nogil: - - cdef cppclass YBValue: - @staticmethod - YBValue* FromInt(int64_t val); - - @staticmethod - YBValue* FromFloat(float val); - - @staticmethod - YBValue* FromDouble(double val); - - @staticmethod - YBValue* FromBool(c_bool val); - - @staticmethod - YBValue* CopyString(const Slice& s); - - -cdef extern from "yb/client/client.h" namespace "yb::client" nogil: - - # Omitted YBClient::ReplicaSelection enum - - cdef cppclass YBClient: - - Status DeleteTable(const string& table_name) - Status OpenTable(const string& table_name, - shared_ptr[YBTable]* table) - Status GetTableSchema(const string& table_name, YBSchema* schema) - - YBTableCreator* NewTableCreator() - Status IsCreateTableInProgress(const string& table_name, - c_bool* create_in_progress) - - c_bool IsMultiMaster() - - Status ListTables(vector[string]* tables) - Status ListTables(vector[string]* tables, const string& filter) - - Status TableExists(const string& table_name, c_bool* exists) - - YBTableAlterer* NewTableAlterer() - Status IsAlterTableInProgress(const string& table_name, - c_bool* alter_in_progress) - - shared_ptr[YBSession] NewSession() - - cdef cppclass YBClientBuilder: - YBClientBuilder() - YBClientBuilder& master_server_addrs(const vector[string]& addrs) - YBClientBuilder& add_master_server_addr(const string& addr) - - YBClientBuilder& default_admin_operation_timeout( - const MonoDelta& timeout) - - YBClientBuilder& default_rpc_timeout(const MonoDelta& timeout) - - Status Build(shared_ptr[YBClient]* client) - - cdef cppclass YBTableCreator: - YBTableCreator& table_name(string& name) - YBTableCreator& schema(YBSchema* schema) - YBTableCreator& split_keys(vector[string]& keys) - YBTableCreator& num_replicas(int n_replicas) - YBTableCreator& wait(c_bool wait) - - Status Create() - - cdef cppclass YBTableAlterer: - # The name of the existing table to alter - YBTableAlterer& table_name(string& name) - - YBTableAlterer& rename_table(string& name) - - YBTableAlterer& add_column(string& name, DataType type, - const void *default_value) - YBTableAlterer& add_column(string& name, DataType type, - const void *default_value, - YBColumnStorageAttributes attr) - - YBTableAlterer& add_nullable_column(string& name, DataType type) - - YBTableAlterer& drop_column(string& name) - - YBTableAlterer& rename_column(string& old_name, string& new_name) - - YBTableAlterer& wait(c_bool wait) - - Status Alter() - - # Instances of YBTable are not directly instantiated by users of the - # client. - cdef cppclass YBTable: - - string& name() - YBSchema& schema() - - YBInsert* NewInsert() - YBUpdate* NewUpdate() - YBDelete* NewDelete() - - YBPredicate* NewComparisonPredicate(const Slice& col_name, - ComparisonOp op, - YBValue* value); - - YBClient* client() - # const PartitionSchema& partition_schema() - - enum FlushMode" yb::client::YBSession::FlushMode": - FlushMode_AutoSync " yb::client::YBSession::AUTO_FLUSH_SYNC" - FlushMode_AutoBackground " yb::client::YBSession::AUTO_FLUSH_BACKGROUND" - FlushMode_Manual " yb::client::YBSession::MANUAL_FLUSH" - - cdef cppclass YBSession: - - Status SetFlushMode(FlushMode m) - - void SetMutationBufferSpace(size_t size) - void SetTimeoutMillis(int millis) - - void SetPriority(int priority) - - Status Apply(YBWriteOperation* write_op) - Status Apply(YBInsert* write_op) - Status Apply(YBUpdate* write_op) - Status Apply(YBDelete* write_op) - - # This is thread-safe - Status Flush() - - # TODO: Will need to decide on a strategy for exposing the session's - # async API to Python - - # Status ApplyAsync(YBWriteOperation* write_op, - # YBStatusCallback cb) - # Status ApplyAsync(YBInsert* write_op, - # YBStatusCallback cb) - # Status ApplyAsync(YBUpdate* write_op, - # YBStatusCallback cb) - # Status ApplyAsync(YBDelete* write_op, - # YBStatusCallback cb) - # void FlushAsync(YBStatusCallback& cb) - - - Status Close() - c_bool HasPendingOperations() - int CountBufferedOperations() - - int CountPendingErrors() - void GetPendingErrors(vector[C_YBError*]* errors, c_bool* overflowed) - - YBClient* client() - - enum ReadMode" yb::client::YBScanner::ReadMode": - READ_LATEST " yb::client::YBScanner::READ_LATEST" - READ_AT_SNAPSHOT " yb::client::YBScanner::READ_AT_SNAPSHOT" - - cdef cppclass YBScanner: - YBScanner(YBTable* table) - - Status AddConjunctPredicate(YBPredicate* pred) - - Status Open() - void Close() - - c_bool HasMoreRows() - Status NextBatch(vector[YBRowResult]* rows) - Status SetBatchSizeBytes(uint32_t batch_size) - - # Pending definition of ReplicaSelection enum - # Status SetSelection(ReplicaSelection selection) - - Status SetReadMode(ReadMode read_mode) - Status SetSnapshot(uint64_t snapshot_timestamp_micros) - Status SetTimeoutMillis(int millis) - Status SetFaultTolerant() - - string ToString() - - cdef cppclass C_YBError " yb::client::YBError": - - Status& status() - - YBWriteOperation& failed_op() - YBWriteOperation* release_failed_op() - - c_bool was_possibly_successful() diff --git a/python/kudu/schema.pxd b/python/kudu/schema.pxd deleted file mode 100644 index 9ec11eedacfd..000000000000 --- a/python/kudu/schema.pxd +++ /dev/null @@ -1,59 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from libcpp.map cimport map - -from libyb_client cimport * - - -cdef class KuduType(object): - cdef readonly: - DataType type - - -cdef class ColumnSchema: - """ - Wraps a Kudu client ColumnSchema object - """ - cdef: - YBColumnSchema* schema - KuduType _type - - -cdef class ColumnSpec: - cdef: - YBColumnSpec* spec - - -cdef class SchemaBuilder: - cdef: - YBSchemaBuilder builder - - -cdef class Schema: - cdef: - const YBSchema* schema - object parent - bint own_schema - map[string, int] _col_mapping - bint _mapping_initialized - - cdef int get_loc(self, name) except -1 - - cdef inline DataType loc_type(self, int i): - return self.schema.Column(i).type() diff --git a/python/kudu/schema.pyx b/python/kudu/schema.pyx deleted file mode 100644 index 1647d7d248eb..000000000000 --- a/python/kudu/schema.pyx +++ /dev/null @@ -1,560 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -# distutils: language = c++ -# cython: embedsignature = True - -from cython.operator cimport dereference as deref - -from kudu.compat import tobytes, frombytes -from kudu.schema cimport * -from kudu.errors cimport check_status - -import six - -from . import util - -BOOL = YB_BOOL -STRING = YB_STRING - -INT8 = YB_INT8 -INT16 = YB_INT16 -INT32 = YB_INT32 -INT64 = YB_INT64 - -FLOAT = YB_FLOAT -DOUBLE = YB_DOUBLE - -TIMESTAMP = YB_TIMESTAMP -BINARY = YB_BINARY - - -cdef dict _reverse_dict(d): - return dict((v, k) for k, v in d.items()) - - -# CompressionType enums -COMPRESSION_DEFAULT = CompressionType_DEFAULT -COMPRESSION_NONE = CompressionType_NONE -COMPRESSION_SNAPPY = CompressionType_SNAPPY -COMPRESSION_LZ4 = CompressionType_LZ4 -COMPRESSION_ZLIB = CompressionType_ZLIB - -cdef dict _compression_types = { - 'default': COMPRESSION_DEFAULT, - 'none': COMPRESSION_NONE, - 'snappy': COMPRESSION_SNAPPY, - 'lz4': COMPRESSION_LZ4, - 'zlib': COMPRESSION_ZLIB, -} - -cdef dict _compression_type_to_name = _reverse_dict(_compression_types) - - -# EncodingType enums -ENCODING_AUTO = EncodingType_AUTO -ENCODING_PLAIN = EncodingType_PLAIN -ENCODING_PREFIX = EncodingType_PREFIX -ENCODING_GROUP_VARINT = EncodingType_GROUP_VARINT -ENCODING_RLE = EncodingType_RLE - -cdef dict _encoding_types = { - 'auto': ENCODING_AUTO, - 'plain': ENCODING_PLAIN, - 'prefix': ENCODING_PREFIX, - 'group_varint': ENCODING_GROUP_VARINT, - 'rle': ENCODING_RLE, -} - -cdef dict _encoding_type_to_name = _reverse_dict(_encoding_types) - - -cdef class KuduType(object): - - """ - Usability wrapper for Kudu data type enum - """ - - def __cinit__(self, DataType type): - self.type = type - - property name: - - def __get__(self): - return _type_names[self.type] - - def __repr__(self): - return 'KuduType({0})'.format(self.name) - - -int8 = KuduType(YB_INT8) -int16 = KuduType(YB_INT16) -int32 = KuduType(YB_INT32) -int64 = KuduType(YB_INT64) -string_ = KuduType(YB_STRING) -bool_ = KuduType(YB_BOOL) -float_ = KuduType(YB_FLOAT) -double_ = KuduType(YB_DOUBLE) -binary = KuduType(YB_BINARY) -timestamp = KuduType(YB_TIMESTAMP) - - -cdef dict _type_names = { - INT8: 'int8', - INT16: 'int16', - INT32: 'int32', - INT64: 'int64', - STRING: 'string', - BOOL: 'bool', - FLOAT: 'float', - DOUBLE: 'double', - BINARY: 'binary', - TIMESTAMP: 'timestamp' -} - - -cdef dict _type_name_to_number = _reverse_dict(_type_names) - -cdef dict _type_to_obj = { - INT8: int8, - INT16: int16, - INT32: int32, - INT64: int64, - STRING: string_, - BOOL: bool_, - FLOAT: float_, - DOUBLE: double_, - BINARY: binary, - TIMESTAMP: timestamp -} - - -cdef KuduType to_data_type(object obj): - if isinstance(obj, KuduType): - return obj - elif isinstance(obj, six.string_types): - return _type_to_obj[_type_name_to_number[obj]] - elif obj in _type_to_obj: - return _type_to_obj[obj] - else: - raise ValueError('Invalid type: {0}'.format(obj)) - - -cdef class ColumnSchema: - """ - Wraps a Kudu client ColumnSchema object. Use schema.at(i) or schema[i] to - construct one. - """ - - def __cinit__(self): - self.schema = NULL - self._type = None - - def __dealloc__(self): - if self.schema is not NULL: - del self.schema - - property name: - def __get__(self): - return frombytes(self.schema.name()) - - property type: - def __get__(self): - if self._type is None: - self._type = _type_to_obj[self.schema.type()] - return self._type - - property nullable: - def __get__(self): - return self.schema.is_nullable() - - def equals(self, other): - if not isinstance(other, ColumnSchema): - return False - return self.schema.Equals(deref(( other).schema)) - - def __repr__(self): - return ('ColumnSchema(name=%s, type=%s, nullable=%s)' - % (self.name, self.type.name, - self.nullable)) - - -#---------------------------------------------------------------------- - -cdef class ColumnSpec: - - """ - Helper class for configuring a column's settings while using the - SchemaBuilder. - """ - - def type(self, type_): - self.spec.Type(to_data_type(type_).type) - return self - - def default(self, value): - """ - Set a default value for the column - """ - raise NotImplementedError - - def clear_default(self): - """ - Remove a default value set. - """ - raise NotImplementedError - - def compression(self, compression): - """ - Set the compression type - - Parameters - ---------- - compression : string or int - One of {'default', 'none', 'snappy', 'lz4', 'zlib'} - Or see kudu.COMPRESSION_* constants - - Returns - ------- - self - """ - cdef CompressionType type - if isinstance(compression, int): - # todo: validation - type = compression - else: - if compression is None: - type = CompressionType_NONE - else: - try: - type = _compression_types[compression.lower()] - except KeyError: - raise ValueError('Invalid compression type: {0}' - .format(compression)) - - self.spec.Compression(type) - return self - - def encoding(self, encoding): - """ - Set the encoding type - - Parameters - ---------- - encoding : string or int - One of {'auto', 'plain', 'prefix', 'group_varint', 'rle'} - Or see kudu.ENCODING_* constants - - Returns - ------- - self - """ - cdef EncodingType type - if isinstance(encoding, six.string_types): - try: - type = _encoding_types[encoding.lower()] - except KeyError: - raise ValueError('Invalid encoding type: {0}' - .format(encoding)) - else: - # todo: validation - type = encoding - - self.spec.Encoding(type) - return self - - def primary_key(self): - """ - Make this column a primary key. If you use this method, it will be the - only primary key. Otherwise see set_primary_keys method on - SchemaBuilder. - - Returns - ------- - self - """ - self.spec.PrimaryKey() - return self - - def nullable(self, bint is_nullable=True): - """ - Set nullable (True) or not nullable (False) - - Parameters - ---------- - is_nullable : boolean, default True - - Returns - ------- - self - """ - if is_nullable: - self.spec.Nullable() - else: - self.spec.NotNull() - return self - - def rename(self, new_name): - """ - Change the column name. - - TODO: Not implemented for table creation - """ - self.spec.RenameTo(new_name) - return self - - -cdef class SchemaBuilder: - - def add_column(self, name, type_=None, nullable=None, compression=None, - encoding=None, primary_key=False): - """ - Add a new column to the schema. Returns a ColumnSpec object for further - configuration and use in a fluid programming style. - - Parameters - ---------- - name : string - type_ : string or KuduType - Data type e.g. 'int32' or kudu.int32 - nullable : boolean, default None - New columns are nullable by default. Set boolean value for explicit - nullable / not-nullable - compression : string or int - One of {'default', 'none', 'snappy', 'lz4', 'zlib'} - Or see kudu.COMPRESSION_* constants - encoding : string or int - One of {'auto', 'plain', 'prefix', 'group_varint', 'rle'} - Or see kudu.ENCODING_* constants - primary_key : boolean, default False - Use this column as the table primary key - - Examples - -------- - (builder.add_column('foo') - .nullable(True) - .compression('lz4')) - - Returns - ------- - spec : ColumnSpec - """ - cdef: - ColumnSpec result = ColumnSpec() - string c_name = tobytes(name) - - result.spec = self.builder.AddColumn(c_name) - - if type_ is not None: - result.type(type_) - - if nullable is not None: - result.nullable(nullable) - - if compression is not None: - result.compression(compression) - - if encoding is not None: - result.encoding(encoding) - - if primary_key: - result.primary_key() - - return result - - def set_primary_keys(self, key_names): - """ - Set indicated columns (by name) to be the primary keys of the table - schema - - Parameters - ---------- - key_names : list of Python strings - - Returns - ------- - None - """ - cdef: - vector[string] key_col_names - - for name in key_names: - key_col_names.push_back(tobytes(name)) - - self.builder.SetPrimaryKey(key_col_names) - - def build(self): - """ - Creates an immutable Schema object after the user has finished adding - and onfiguring columns - - Returns - ------- - schema : Schema - """ - cdef Schema result = Schema() - cdef YBSchema* schema = new YBSchema() - check_status(self.builder.Build(schema)) - - result.schema = schema - return result - - -cdef class Schema: - - """ - Container for a Kudu table schema. Obtain from Table instances or create - new ones using kudu.SchemaBuilder - """ - - def __cinit__(self): - # Users should not call this directly - self.schema = NULL - self.own_schema = 1 - self._col_mapping.clear() - self._mapping_initialized = 0 - - def __dealloc__(self): - if self.schema is not NULL and self.own_schema: - del self.schema - - property names: - - def __get__(self): - result = [] - for i in range(self.schema.num_columns()): - name = frombytes(self.schema.Column(i).name()) - result.append(name) - - return result - - def __repr__(self): - # Got to be careful with huge schemas, maybe some kind of summary repr - # when more than 20-30 columns? - buf = six.StringIO() - - col_names = self.names - space = 2 + max(len(x) for x in col_names) - - for i in range(len(self)): - col = self.at(i) - not_null = '' if col.nullable else ' NOT NULL' - - buf.write('\n{0}{1}{2}' - .format(col.name.ljust(space), - col.type.name, not_null)) - - pk_string = ', '.join(col_names[i] for i in self.primary_key_indices()) - buf.write('\nPRIMARY KEY ({0})'.format(pk_string)) - - return "kudu.Schema {{{0}\n}}".format(util.indent(buf.getvalue(), 2)) - - def __len__(self): - return self.schema.num_columns() - - def __getitem__(self, key): - if isinstance(key, six.string_types): - key = self.get_loc(key) - - if key < 0: - key += len(self) - return self.at(key) - - def equals(self, Schema other): - """ - Returns True if the table schemas are equal - """ - return self.schema.Equals(deref(other.schema)) - - cdef int get_loc(self, name) except -1: - if not self._mapping_initialized: - for i in range(self.schema.num_columns()): - self._col_mapping[self.schema.Column(i).name()] = i - self._mapping_initialized = 1 - - name = tobytes(name) - - # TODO: std::map is slightly verbose and inefficient here (O(lg n) - # lookups), may consider replacing with a better / different hash table - # should it become a performance bottleneck - cdef map[string, int].iterator it = self._col_mapping.find(name) - if it == self._col_mapping.end(): - raise KeyError(name) - return self._col_mapping[name] - - def at(self, size_t i): - """ - Return the ColumnSchema for a column index. Analogous to schema[i]. - - Returns - ------- - col_schema : ColumnSchema - """ - cdef ColumnSchema result = ColumnSchema() - - if i < 0 or i >= self.schema.num_columns(): - raise IndexError('Column index {0} is not in range' - .format(i)) - - result.schema = new YBColumnSchema(self.schema.Column(i)) - - return result - - def primary_key_indices(self): - """ - Return the indices of the columns used as primary keys - - Returns - ------- - key_indices : list[int] - """ - cdef: - vector[int] indices - size_t i - - self.schema.GetPrimaryKeyColumnIndexes(&indices) - - result = [] - for i in range(indices.size()): - result.append(indices[i]) - return result - - def primary_keys(self): - """ - Return the names of the columns used as primary keys - - Returns - ------- - key_names : list[str] - """ - indices = self.primary_key_indices() - return [self.at(i).name for i in indices] diff --git a/python/kudu/tests/__init__.py b/python/kudu/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py deleted file mode 100644 index 35d28a6176a0..000000000000 --- a/python/kudu/tests/common.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from __future__ import division - -import json -import fnmatch -import os -import shutil -import subprocess -import tempfile -import time - -import kudu - - -class YBTestBase(object): - - """ - Base test class that will start a configurable number of master and - tablet servers. - """ - - BASE_PORT = 37000 - NUM_TABLET_SERVERS = 3 - - @classmethod - def start_cluster(cls): - local_path = tempfile.mkdtemp(dir=os.getenv("TEST_TMPDIR")) - kudu_build = os.getenv("YB_BUILD") - if not kudu_build: - kudu_build = os.path.join(os.getenv("YB_HOME"), "build", "latest") - bin_path = "{0}/bin".format(kudu_build) - - os.makedirs("{0}/master/".format(local_path)) - os.makedirs("{0}/master/data".format(local_path)) - os.makedirs("{0}/master/logs".format(local_path)) - - path = [ - "{0}/yb-master".format(bin_path), - "-rpc_server_allow_ephemeral_ports", - "-rpc_bind_addresses=0.0.0.0:0", - "-fs_wal_dirs={0}/master/data".format(local_path), - "-fs_data_dirs={0}/master/data".format(local_path), - "-log_dir={0}/master/logs".format(local_path), - "-logtostderr", - "-webserver_port=0", - # Only make one replica so that our tests don't need to worry about - # setting consistency modes. - "-default_num_replicas=1", - "-server_dump_info_path={0}/master/config.json".format(local_path) - ] - - p = subprocess.Popen(path, shell=False) - fid = open("{0}/master/yb-master.pid".format(local_path), "w+") - fid.write("{0}".format(p.pid)) - fid.close() - - # We have to wait for the master to settle before the config file - # appears - config_file = "{0}/master/config.json".format(local_path) - for i in range(30): - if os.path.exists(config_file): - break - time.sleep(0.1 * (i + 1)) - else: - raise Exception("Could not find yb-master config file") - - # If the server was started get the bind port from the config dump - master_config = json.load(open("{0}/master/config.json" - .format(local_path), "r")) - # One master bound on local host - master_port = master_config["bound_rpc_addresses"][0]["port"] - - for m in range(cls.NUM_TABLET_SERVERS): - os.makedirs("{0}/ts/{1}".format(local_path, m)) - os.makedirs("{0}/ts/{1}/logs".format(local_path, m)) - - path = [ - "{0}/yb-tserver".format(bin_path), - "-rpc_server_allow_ephemeral_ports", - "-rpc_bind_addresses=0.0.0.0:0", - "-tserver_master_addrs=127.0.0.1:{0}".format(master_port), - "-webserver_port=0", - "-log_dir={0}/master/logs".format(local_path), - "-logtostderr", - "-fs_data_dirs={0}/ts/{1}/data".format(local_path, m), - "-fs_wal_dirs={0}/ts/{1}/data".format(local_path, m), - ] - p = subprocess.Popen(path, shell=False) - tserver_pid = "{0}/ts/{1}/yb-tserver.pid".format(local_path, m) - fid = open(tserver_pid, "w+") - fid.write("{0}".format(p.pid)) - fid.close() - - return local_path, master_port - - @classmethod - def stop_cluster(cls, path): - for root, dirnames, filenames in os.walk('{0}/..'.format(path)): - for filename in fnmatch.filter(filenames, '*.pid'): - with open(os.path.join(root, filename)) as fid: - a = fid.read() - r = subprocess.Popen(["kill", "{0}".format(a)]) - r.wait() - os.remove(os.path.join(root, filename)) - shutil.rmtree(path, True) - - @classmethod - def setUpClass(cls): - cls.cluster_path, master_port = cls.start_cluster() - time.sleep(1) - - cls.master_host = '127.0.0.1' - cls.master_port = master_port - - cls.client = kudu.connect(cls.master_host, cls.master_port) - - cls.schema = cls.example_schema() - - cls.ex_table = 'example-table' - if cls.client.table_exists(cls.ex_table): - cls.client.delete_table(cls.ex_table) - cls.client.create_table(cls.ex_table, cls.schema) - - @classmethod - def tearDownClass(cls): - cls.stop_cluster(cls.cluster_path) - - @classmethod - def example_schema(cls): - builder = kudu.schema_builder() - builder.add_column('key', kudu.int32, nullable=False) - builder.add_column('int_val', kudu.int32) - builder.add_column('string_val', kudu.string) - builder.set_primary_keys(['key']) - - return builder.build() diff --git a/python/kudu/tests/test_client.py b/python/kudu/tests/test_client.py deleted file mode 100644 index 40769a54400f..000000000000 --- a/python/kudu/tests/test_client.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from kudu.compat import unittest, long -from kudu.tests.common import YBTestBase -import kudu - - -class TestClient(YBTestBase, unittest.TestCase): - - def setUp(self): - pass - - def test_table_basics(self): - table = self.client.table(self.ex_table) - - self.assertEqual(table.name, self.ex_table) - self.assertEqual(table.num_columns, len(self.schema)) - - def test_table_column(self): - table = self.client.table(self.ex_table) - col = table['key'] - - assert col.name == b'key' - assert col.parent is table - - result_repr = repr(col) - expected_repr = ('Column(key, parent={0}, type=int32)' - .format(self.ex_table)) - assert result_repr == expected_repr - - def test_table_schema_retains_reference(self): - import gc - - table = self.client.table(self.ex_table) - schema = table.schema - table = None - - gc.collect() - repr(schema) - - def test_table_exists(self): - self.assertFalse(self.client.table_exists('nonexistent-table')) - self.assertTrue(self.client.table_exists(self.ex_table)) - - def test_list_tables(self): - schema = self.example_schema() - - to_create = ['foo1', 'foo2', 'foo3'] - for name in to_create: - self.client.create_table(name, schema) - - result = self.client.list_tables() - expected = [self.ex_table] + to_create - assert sorted(result) == expected - - result = self.client.list_tables('foo') - assert sorted(result) == to_create - - for name in to_create: - self.client.delete_table(name) - - def test_is_multimaster(self): - assert not self.client.is_multimaster - - def test_delete_table(self): - name = "peekaboo" - self.client.create_table(name, self.schema) - self.client.delete_table(name) - assert not self.client.table_exists(name) - - # Should raise a more meaningful exception at some point - with self.assertRaises(kudu.KuduNotFound): - self.client.delete_table(name) - - def test_table_nonexistent(self): - self.assertRaises(kudu.KuduNotFound, self.client.table, - '__donotexist__') - - def test_insert_nonexistent_field(self): - table = self.client.table(self.ex_table) - op = table.new_insert() - self.assertRaises(KeyError, op.__setitem__, 'doesntexist', 12) - - def test_insert_rows_and_delete(self): - nrows = 100 - table = self.client.table(self.ex_table) - session = self.client.new_session() - for i in range(nrows): - op = table.new_insert() - op['key'] = i - op['int_val'] = i * 2 - op['string_val'] = 'hello_%d' % i - session.apply(op) - - # Cannot apply the same insert twice, C++ client does not indicate an - # error - self.assertRaises(Exception, session.apply, op) - - # synchronous - session.flush() - - scanner = table.scanner().open() - assert len(scanner.read_all_tuples()) == nrows - - # Delete the rows we just wrote - for i in range(nrows): - op = table.new_delete() - op['key'] = i - session.apply(op) - session.flush() - - scanner = table.scanner().open() - assert len(scanner.read_all_tuples()) == 0 - - def test_session_auto_open(self): - table = self.client.table(self.ex_table) - scanner = table.scanner() - result = scanner.read_all_tuples() - assert len(result) == 0 - - def test_session_open_idempotent(self): - table = self.client.table(self.ex_table) - scanner = table.scanner().open().open() - result = scanner.read_all_tuples() - assert len(result) == 0 - - def test_session_flush_modes(self): - self.client.new_session(flush_mode=kudu.FLUSH_MANUAL) - self.client.new_session(flush_mode=kudu.FLUSH_AUTO_SYNC) - - self.client.new_session(flush_mode='manual') - self.client.new_session(flush_mode='sync') - - with self.assertRaises(kudu.KuduNotSupported): - self.client.new_session(flush_mode=kudu.FLUSH_AUTO_BACKGROUND) - - with self.assertRaises(kudu.KuduNotSupported): - self.client.new_session(flush_mode='background') - - with self.assertRaises(ValueError): - self.client.new_session(flush_mode='foo') - - def test_connect_timeouts(self): - # it works! any other way to check - kudu.connect(self.master_host, self.master_port, - admin_timeout_ms=100, - rpc_timeout_ms=100) - - def test_capture_kudu_error(self): - pass - - -class TestMonoDelta(unittest.TestCase): - - def test_empty_ctor(self): - delta = kudu.TimeDelta() - assert repr(delta) == 'kudu.TimeDelta()' - - def test_static_ctors(self): - delta = kudu.timedelta(3.5) - assert delta.to_seconds() == 3.5 - - delta = kudu.timedelta(millis=3500) - assert delta.to_millis() == 3500 - - delta = kudu.timedelta(micros=3500) - assert delta.to_micros() == 3500 - - delta = kudu.timedelta(micros=1000) - assert delta.to_nanos() == long(1000000) - - delta = kudu.timedelta(nanos=3500) - assert delta.to_nanos() == 3500 diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py deleted file mode 100644 index 26cd2f7bcad5..000000000000 --- a/python/kudu/tests/test_scanner.py +++ /dev/null @@ -1,120 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from __future__ import division - -from kudu.compat import unittest -from kudu.tests.common import YBTestBase -import kudu - - -class TestScanner(YBTestBase, unittest.TestCase): - - @classmethod - def setUpClass(cls): - super(TestScanner, cls).setUpClass() - - cls.nrows = 100 - table = cls.client.table(cls.ex_table) - session = cls.client.new_session() - - tuples = [] - for i in range(cls.nrows): - op = table.new_insert() - tup = i, i * 2, 'hello_%d' % i if i % 2 == 0 else None - op['key'] = tup[0] - op['int_val'] = tup[1] - if i % 2 == 0: - op['string_val'] = tup[2] - session.apply(op) - tuples.append(tup) - session.flush() - - cls.table = table - cls.tuples = tuples - - @classmethod - def tearDownClass(cls): - pass - - def setUp(self): - pass - - def test_scan_rows_basic(self): - # Let's scan with no predicates - scanner = self.table.scanner().open() - - tuples = scanner.read_all_tuples() - self.assertEqual(sorted(tuples), self.tuples) - - def test_scan_rows_simple_predicate(self): - key = self.table['key'] - preds = [key >= 20, key <= 49] - - def _read_predicates(preds): - scanner = self.table.scanner() - scanner.add_predicates(preds) - scanner.open() - return scanner.read_all_tuples() - - tuples = _read_predicates(preds) - self.assertEqual(sorted(tuples), self.tuples[20:50]) - - # verify predicates reusable - tuples = _read_predicates(preds) - self.assertEqual(sorted(tuples), self.tuples[20:50]) - - def test_scan_rows_string_predicate(self): - scanner = self.table.scanner() - - sv = self.table['string_val'] - - scanner.add_predicates([sv >= 'hello_20', - sv <= 'hello_22']) - - scanner.set_fault_tolerant() - scanner.open() - - tuples = scanner.read_all_tuples() - - self.assertEqual(sorted(tuples), [(20, 40, 'hello_20'), (22, 44, 'hello_22')]) - - def test_scan_invalid_predicates(self): - scanner = self.table.scanner() - sv = self.table['string_val'] - - with self.assertRaises(TypeError): - scanner.add_predicates([sv >= None]) - - with self.assertRaises(kudu.KuduInvalidArgument): - scanner.add_predicates([sv >= 1]) diff --git a/python/kudu/tests/test_schema.py b/python/kudu/tests/test_schema.py deleted file mode 100644 index 32242d11445f..000000000000 --- a/python/kudu/tests/test_schema.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from __future__ import division - -from kudu.compat import unittest -import kudu - - -class TestSchema(unittest.TestCase): - - def setUp(self): - self.columns = [('one', 'int32', False), - ('two', 'int8', False), - ('three', 'double', True), - ('four', 'string', False)] - - self.primary_keys = ['one', 'two'] - - self.builder = kudu.schema_builder() - for name, typename, nullable in self.columns: - self.builder.add_column(name, typename, nullable=nullable) - - self.builder.set_primary_keys(self.primary_keys) - self.schema = self.builder.build() - - def test_repr(self): - result = repr(self.schema) - for name, _, _ in self.columns: - assert name in result - - assert 'PRIMARY KEY (one, two)' in result - - def test_schema_length(self): - assert len(self.schema) == 4 - - def test_names(self): - assert self.schema.names == ['one', 'two', 'three', 'four'] - - def test_primary_keys(self): - assert self.schema.primary_key_indices() == [0, 1] - assert self.schema.primary_keys() == ['one', 'two'] - - def test_getitem_boundschecking(self): - with self.assertRaises(IndexError): - self.schema[4] - - def test_getitem_wraparound(self): - # wraparound - result = self.schema[-1] - expected = self.schema[3] - - assert result.equals(expected) - - def test_getitem_string(self): - result = self.schema['three'] - expected = self.schema[2] - - assert result.equals(expected) - - with self.assertRaises(KeyError): - self.schema['not_found'] - - def test_schema_equals(self): - assert self.schema.equals(self.schema) - - builder = kudu.schema_builder() - builder.add_column('key', 'int64', nullable=False, primary_key=True) - schema = builder.build() - - assert not self.schema.equals(schema) - - def test_column_equals(self): - assert not self.schema[0].equals(self.schema[1]) - - def test_type(self): - builder = kudu.schema_builder() - (builder.add_column('key') - .type('int32') - .primary_key() - .nullable(False)) - schema = builder.build() - - tp = schema[0].type - assert tp.name == 'int32' - assert tp.type == kudu.schema.INT32 - - def test_compression(self): - builder = kudu.schema_builder() - builder.add_column('key', 'int64', nullable=False) - - foo = builder.add_column('foo', 'string').compression('lz4') - assert foo is not None - - bar = builder.add_column('bar', 'string') - bar.compression(kudu.COMPRESSION_ZLIB) - - with self.assertRaises(ValueError): - bar = builder.add_column('qux', 'string', compression='unknown') - - builder.set_primary_keys(['key']) - builder.build() - - # TODO; The C++ client does not give us an API to see the storage - # attributes of a column - - def test_encoding(self): - builder = kudu.schema_builder() - builder.add_column('key', 'int64', nullable=False) - - foo = builder.add_column('foo', 'string').encoding('rle') - assert foo is not None - - bar = builder.add_column('bar', 'string') - bar.encoding(kudu.ENCODING_PLAIN) - - with self.assertRaises(ValueError): - builder.add_column('qux', 'string', encoding='unknown') - - builder.set_primary_keys(['key']) - builder.build() - # TODO(wesm): The C++ client does not give us an API to see the storage - # attributes of a column - - def test_set_column_spec_pk(self): - builder = kudu.schema_builder() - key = (builder.add_column('key', 'int64', nullable=False) - .primary_key()) - assert key is not None - schema = builder.build() - assert 'key' in schema.primary_keys() - - builder = kudu.schema_builder() - key = (builder.add_column('key', 'int64', nullable=False, - primary_key=True)) - schema = builder.build() - assert 'key' in schema.primary_keys() - - def test_partition_schema(self): - pass - - def test_nullable_not_null(self): - builder = kudu.schema_builder() - (builder.add_column('key', 'int64', nullable=False) - .primary_key()) - - builder.add_column('data1', 'double').nullable(True) - builder.add_column('data2', 'double').nullable(False) - builder.add_column('data3', 'double', nullable=True) - builder.add_column('data4', 'double', nullable=False) - - schema = builder.build() - - assert not schema[0].nullable - assert schema[1].nullable - assert not schema[2].nullable - - assert schema[3].nullable - assert not schema[4].nullable - - def test_default_value(self): - pass - - def test_column_schema_repr(self): - result = repr(self.schema[0]) - expected = 'ColumnSchema(name=one, type=int32, nullable=False)' - self.assertEqual(result, expected) diff --git a/python/kudu/util.py b/python/kudu/util.py deleted file mode 100644 index f4f783d10ac3..000000000000 --- a/python/kudu/util.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - - -def indent(text, spaces): - block = ' ' * spaces - return '\n'.join(block + x for x in text.split('\n')) diff --git a/python/requirements.txt b/python/requirements.txt deleted file mode 100644 index ef646fa036df..000000000000 --- a/python/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest -numpy>=1.7.0 -cython >= 0.21 -setuptools >= 0.8 -six -unittest2 diff --git a/python/setup.cfg b/python/setup.cfg deleted file mode 100644 index 967bc1922688..000000000000 --- a/python/setup.cfg +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -[aliases] -test=pytest diff --git a/python/setup.py b/python/setup.py deleted file mode 100644 index e1066213c3db..000000000000 --- a/python/setup.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# - -from Cython.Distutils import build_ext -from Cython.Build import cythonize -import Cython - -import sys -from setuptools import setup -from distutils.command.clean import clean as _clean -from distutils.extension import Extension -import os - -if Cython.__version__ < '0.19.1': - raise Exception('Please upgrade to Cython 0.19.1 or newer') - -MAJOR = 0 -MINOR = 1 -MICRO = 1 -VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) -ISRELEASED = True - -setup_dir = os.path.abspath(os.path.dirname(__file__)) - - -def write_version_py(filename=os.path.join(setup_dir, 'kudu/version.py')): - version = VERSION - if not ISRELEASED: - version += '.dev' - - a = open(filename, 'w') - file_content = "\n".join(["", - "# THIS FILE IS GENERATED FROM SETUP.PY", - "version = '%(version)s'", - "isrelease = '%(isrelease)s'"]) - - a.write(file_content % {'version': VERSION, - 'isrelease': str(ISRELEASED)}) - a.close() - - -class clean(_clean): - def run(self): - _clean.run(self) - for x in ['kudu/client.cpp', 'kudu/schema.cpp', - 'kudu/errors.cpp']: - try: - os.remove(x) - except OSError: - pass - - -# If we're in the context of the Kudu git repository, build against the -# latest in-tree build artifacts -if 'YB_HOME' in os.environ: - yb_home = os.environ['YB_HOME'] - sys.stderr.write("Using YB_HOME directory: %s\n" % (yb_home,)) - if not os.path.isdir(yb_home): - sys.stderr.write("%s is not a valid YB_HOME directory" % (yb_home,)) - sys.exit(1) - - kudu_include_dirs = [os.path.join(yb_home, 'src')] - - if 'YB_BUILD' in os.environ: - kudu_build = os.environ['YB_BUILD'] - sys.stderr.write("Using YB_BUILD directory: %s\n" % (kudu_build,)) - else: - kudu_build = os.path.join(yb_home, 'build', 'latest') - sys.stderr.write("Using inferred YB_BUILD directory: %s/\n" % (kudu_build,)) - if not os.path.isdir(kudu_build): - sys.stderr.write("%s is not a valid YB_BUILD directory" % (kudu_build,)) - sys.exit(1) - - kudu_include_dirs.append(os.path.join(kudu_build, 'src')) - kudu_lib_dir = os.path.join(kudu_build, 'lib', 'exported') -else: - if os.path.exists("/usr/local/include/kudu"): - prefix = "/usr/local" - elif os.path.exists("/usr/include/kudu"): - prefix = "/usr" - else: - sys.stderr.write("Cannot find installed kudu client.\n") - sys.exit(1) - sys.stderr.write("Building from system prefix {0}\n".format(prefix)) - kudu_include_dirs = [prefix + "/include"] - kudu_lib_dir = prefix + "/lib" - -INCLUDE_PATHS = kudu_include_dirs -LIBRARY_DIRS = [kudu_lib_dir] -RT_LIBRARY_DIRS = LIBRARY_DIRS - -ext_submodules = ['client', 'errors', 'schema'] - -extensions = [] - -for submodule_name in ext_submodules: - ext = Extension('kudu.{0}'.format(submodule_name), - ['kudu/{0}.pyx'.format(submodule_name)], - libraries=['yb_client'], - # Disable the 'new' gcc5 ABI; see the top-level - # CMakeLists.txt for details. - define_macros=[('_GLIBCXX_USE_CXX11_ABI', '0')], - include_dirs=INCLUDE_PATHS, - library_dirs=LIBRARY_DIRS, - runtime_library_dirs=RT_LIBRARY_DIRS) - extensions.append(ext) - -extensions = cythonize(extensions) - -write_version_py() - -LONG_DESCRIPTION = open(os.path.join(setup_dir, "README.md")).read() -DESCRIPTION = "Python interface to the Apache Kudu (incubating) C++ Client API" - -CLASSIFIERS = [ - 'Development Status :: 3 - Alpha', - 'Environment :: Console', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Cython' -] - -URL = 'http://getkudu.io' - -setup( - name="kudu-python", - packages=['kudu', 'kudu.tests'], - version=VERSION, - package_data={'kudu': ['*.pxd', '*.pyx']}, - ext_modules=extensions, - cmdclass={ - 'clean': clean, - 'build_ext': build_ext - }, - setup_requires=['pytest-runner'], - tests_require=['pytest'], - install_requires=['cython >= 0.21'], - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - license='Apache License, Version 2.0', - classifiers=CLASSIFIERS, - maintainer="Apache Kudu (incubating) team", - maintainer_email="dev@kudu.incubator.apache.org", - url=URL, - test_suite="kudu.tests" -)