diff --git a/bin/yugabyted b/bin/yugabyted index 27fd7ae92892..6f31b5e76cbe 100755 --- a/bin/yugabyted +++ b/bin/yugabyted @@ -2035,7 +2035,7 @@ class ControlScript(object): placementInfoOfEveryNode = {} for node in dictOfAllNodes: - hostname = re.split('[-:]', node)[0] + hostname = node.split(":")[0] cloudLocationOfHost = "{}.{}.{}".format(dictOfAllNodes[node]["cloud"], dictOfAllNodes[node]["region"], dictOfAllNodes[node]["zone"]) placementInfoOfEveryNode[hostname] = cloudLocationOfHost @@ -4156,7 +4156,12 @@ class YsqlProxy(object): env_var = os.environ.copy() env_var["PGUSER"] = self.username shell = subprocess.Popen(cmd, env=env_var) - shell.communicate() + while True: + try: + shell.communicate() + except KeyboardInterrupt: + continue + break # Checks if db exists. # Note that this will return false if ysqlsh can't connect, even if db exists. @@ -4270,7 +4275,12 @@ class YcqlProxy(object): if self.keyspace is not None: cmd.extend(["-k", self.keyspace]) shell = subprocess.Popen(cmd) - shell.communicate() + while True: + try: + shell.communicate() + except KeyboardInterrupt: + continue + break # Check user exists # Note that this will return false if ycqlsh can't connect, even if user exists. diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index a9ad1144a27f..f69c6c11fa45 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -2171,10 +2171,13 @@ run_shellcheck() { build-support/common-build-env.sh build-support/common-cli-env.sh build-support/common-test-env.sh + build-support/jenkins/common-lto.sh build-support/compiler-wrappers/compiler-wrapper.sh build-support/find_linuxbrew.sh - build-support/jenkins/build-and-test.sh + build-support/jenkins/build.sh + build-support/jenkins/test.sh build-support/jenkins/yb-jenkins-build.sh + build-support/jenkins/yb-jenkins-test.sh build-support/run-test.sh yb_build.sh ) diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build.sh similarity index 72% rename from build-support/jenkins/build-and-test.sh rename to build-support/jenkins/build.sh index 14bad957c1fa..17b8761e21c7 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build.sh @@ -131,13 +131,6 @@ build_cpp_code() { set_yb_src_root "$old_yb_src_root" } -cleanup() { - if [[ -n ${BUILD_ROOT:-} && ${DONT_DELETE_BUILD_ROOT} == "0" ]]; then - log "Running the script to clean up build artifacts..." - "$YB_BUILD_SUPPORT_DIR/jenkins/post-build-clean.sh" - fi -} - # ================================================================================================= # Main script # ================================================================================================= @@ -159,64 +152,8 @@ log "Removing old JSON-based test report files" activate_virtualenv set_pythonpath -# We change YB_RUN_JAVA_TEST_METHODS_SEPARATELY in a subshell in a few places and that is OK. -# shellcheck disable=SC2031 -export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 - -export TSAN_OPTIONS="" - -if is_mac; then - # This is needed to make sure we're using Homebrew-installed CMake on Mac OS X. - export PATH=/usr/local/bin:$PATH -fi - -# gather core dumps -ulimit -c unlimited - -detect_architecture - -BUILD_TYPE=${BUILD_TYPE:-debug} -build_type=$BUILD_TYPE -normalize_build_type -readonly build_type - -BUILD_TYPE=$build_type -readonly BUILD_TYPE -export BUILD_TYPE - -export YB_USE_NINJA=1 - -set_cmake_build_type_and_compiler_type - -if [[ ${YB_DOWNLOAD_THIRDPARTY:-auto} == "auto" ]]; then - log "Setting YB_DOWNLOAD_THIRDPARTY=1 automatically" - export YB_DOWNLOAD_THIRDPARTY=1 -fi -log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY" - -# This is normally done in set_build_root, but we need to decide earlier because this is factored -# into the decision of whether to use LTO. -decide_whether_to_use_linuxbrew - -if [[ -z ${YB_LINKING_TYPE:-} ]]; then - if ! is_mac && [[ - ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ && - ${BUILD_TYPE} == "release" - ]]; then - export YB_LINKING_TYPE=full-lto - else - export YB_LINKING_TYPE=dynamic - fi - log "Automatically decided to set YB_LINKING_TYPE to ${YB_LINKING_TYPE} based on:" \ - "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}," \ - "BUILD_TYPE=${BUILD_TYPE}," \ - "YB_USE_LINUXBREW=${YB_USE_LINUXBREW}," \ - "YB_LINUXBREW_DIR=${YB_LINUXBREW_DIR:-undefined}." -else - log "YB_LINKING_TYPE is already set to ${YB_LINKING_TYPE}" -fi -log "YB_LINKING_TYPE=${YB_LINKING_TYPE}" -export YB_LINKING_TYPE +# shellcheck source=build-support/jenkins/common-lto.sh +. "${BASH_SOURCE%/*}/common-lto.sh" # ------------------------------------------------------------------------------------------------- # Build root setup and build directory cleanup @@ -348,17 +285,6 @@ fi YB_SKIP_CPP_COMPILATION=${YB_SKIP_CPP_COMPILATION:-0} YB_COMPILE_ONLY=${YB_COMPILE_ONLY:-0} -CTEST_OUTPUT_PATH="${BUILD_ROOT}"/ctest.log -CTEST_FULL_OUTPUT_PATH="${BUILD_ROOT}"/ctest-full.log - -TEST_LOG_DIR="${BUILD_ROOT}/test-logs" - -# If we're running inside Jenkins (the BUILD_ID is set), then install an exit handler which will -# clean up all of our build results. -if is_jenkins; then - trap cleanup EXIT -fi - configure_remote_compilation export NO_REBUILD_THIRDPARTY=1 @@ -367,7 +293,6 @@ THIRDPARTY_BIN=$YB_SRC_ROOT/thirdparty/installed/bin export PPROF_PATH=$THIRDPARTY_BIN/pprof # Configure the build -# cd "$BUILD_ROOT" @@ -489,31 +414,6 @@ if [[ ${BUILD_TYPE} != "tsan" ]]; then fi fi -# ------------------------------------------------------------------------------------------------- -# Dependency graph analysis allowing to determine what tests to run. -# ------------------------------------------------------------------------------------------------- - -if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then - if ! ( set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ - --build-root "${BUILD_ROOT}" \ - self-test \ - --rebuild-graph ); then - # Trying to diagnose this error: - # https://gist.githubusercontent.com/mbautin/c5c6f14714f7655c10620d8e658e1f5b/raw - log "dependency_graph.py failed, listing all pb.{h,cc} files in the build directory" - ( set -x; find "$BUILD_ROOT" -name "*.pb.h" -or -name "*.pb.cc" ) - fatal "Dependency graph construction failed" - fi -fi - -# Save the current HEAD commit in case we build Java below and add a new commit. This is used for -# the following purposes: -# - So we can upload the release under the correct commit, from Jenkins, to then be picked up from -# itest, from the snapshots bucket. -# - For picking up the changeset corresponding the the current diff being tested and detecting what -# tests to run in Phabricator builds. If we just diff with origin/master, we'll always pick up -# pom.xml changes we've just made, forcing us to always run Java tests. current_git_commit=$(git rev-parse HEAD) # ------------------------------------------------------------------------------------------------- @@ -703,126 +603,4 @@ else time "${YB_SRC_ROOT}/yb_build.sh" "${BUILD_TYPE}" --build-yugabyted-ui --skip-java fi -# ------------------------------------------------------------------------------------------------- -# Run tests, either on Spark or locally. -# If YB_COMPILE_ONLY is set to 1, we skip running all tests (Java and C++). - -set_sanitizer_runtime_options - -# To reduce Jenkins archive size, let's gzip Java logs and delete per-test-method logs in case -# of no test failures. -export YB_GZIP_PER_TEST_METHOD_LOGS=1 -export YB_GZIP_TEST_LOGS=1 -export YB_DELETE_SUCCESSFUL_PER_TEST_METHOD_LOGS=1 - -if [[ ${YB_COMPILE_ONLY} != "1" ]]; then - if spark_available; then - if [[ ${YB_BUILD_CPP} == "1" || ${YB_BUILD_JAVA} == "1" ]]; then - log "Will run tests on Spark" - run_tests_extra_args=() - if [[ ${YB_BUILD_JAVA} == "1" ]]; then - run_tests_extra_args+=( "--java" ) - fi - if [[ ${YB_BUILD_CPP} == "1" ]]; then - run_tests_extra_args+=( "--cpp" ) - fi - if [[ ${YB_RUN_AFFECTED_TESTS_ONLY} == "1" ]]; then - test_conf_path="${BUILD_ROOT}/test_conf.json" - # YB_GIT_COMMIT_FOR_DETECTING_TESTS allows overriding the commit to use to detect the set - # of tests to run. Useful when testing this script. - ( - set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ - --build-root "${BUILD_ROOT}" \ - --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \ - --output-test-config "${test_conf_path}" \ - affected - ) - run_tests_extra_args+=( "--test_conf" "${test_conf_path}" ) - unset test_conf_path - fi - if is_linux || (is_mac && ! is_src_root_on_nfs); then - log "Will create an archive for Spark workers with all the code instead of using NFS." - run_tests_extra_args+=( "--send_archive_to_workers" ) - fi - # Workers use /private path, which caused mis-match when check is done by yb_dist_tests that - # YB_MVN_LOCAL_REPO is in source tree. So unsetting value here to allow default. - if is_mac; then - unset YB_MVN_LOCAL_REPO - fi - - NUM_REPETITIONS="${YB_NUM_REPETITIONS:-1}" - log "NUM_REPETITIONS is set to ${NUM_REPETITIONS}" - if [[ ${NUM_REPETITIONS} -gt 1 ]]; then - log "Repeating each test ${NUM_REPETITIONS} times" - run_tests_extra_args+=( "--num_repetitions" "${NUM_REPETITIONS}" ) - fi - - set +u # because extra_args can be empty - if ! run_tests_on_spark "${run_tests_extra_args[@]}"; then - set -u - EXIT_STATUS=1 - FAILURES+=$'Distributed tests on Spark (C++ and/or Java) failed\n' - log "Some tests that were run on Spark failed" - fi - set -u - unset extra_args - else - log "Neither C++ or Java tests are enabled, nothing to run on Spark." - fi - else - # A single-node way of running tests (without Spark). - - if [[ ${YB_BUILD_CPP} == "1" ]]; then - log "Run C++ tests in a non-distributed way" - export GTEST_OUTPUT="xml:${TEST_LOG_DIR}/" # Enable JUnit-compatible XML output. - - if ! spark_available; then - log "Did not find Spark on the system, falling back to a ctest-based way of running tests" - set +e - # We don't double-quote EXTRA_TEST_FLAGS on purpose, to allow specifying multiple flags. - # shellcheck disable=SC2086 - time ctest "-j${NUM_PARALLEL_TESTS}" ${EXTRA_TEST_FLAGS:-} \ - --output-log "${CTEST_FULL_OUTPUT_PATH}" \ - --output-on-failure 2>&1 | tee "${CTEST_OUTPUT_PATH}" - ctest_exit_code=$? - set -e - if [[ $ctest_exit_code -ne 0 ]]; then - EXIT_STATUS=1 - FAILURES+=$'C++ tests failed with exit code ${ctest_exit_code}\n' - fi - fi - log "Finished running C++ tests (see timing information above)" - fi - - if [[ ${YB_BUILD_JAVA} == "1" ]]; then - set_test_invocation_id - log "Running Java tests in a non-distributed way" - if ! time run_all_java_test_methods_separately; then - EXIT_STATUS=1 - FAILURES+=$'Java tests failed\n' - fi - log "Finished running Java tests (see timing information above)" - # shellcheck disable=SC2119 - kill_stuck_processes - fi - fi -fi - -# Finished running tests. -remove_latest_symlink - -log "Aggregating test reports" -"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ - --yb-src-root "${YB_SRC_ROOT}" \ - --output-dir "${YB_SRC_ROOT}" \ - --build-type "${build_type}" \ - --compiler-type "${YB_COMPILER_TYPE}" \ - --build-root "${BUILD_ROOT}" - -if [[ -n ${FAILURES} ]]; then - heading "Failure summary" - echo >&2 "${FAILURES}" -fi - exit ${EXIT_STATUS} diff --git a/build-support/jenkins/common-lto.sh b/build-support/jenkins/common-lto.sh new file mode 100755 index 000000000000..46db94a819d4 --- /dev/null +++ b/build-support/jenkins/common-lto.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +# This script is invoked from the Jenkins builds to build YB and run all the unit tests. +# +# Environment variables may be used to customize operation: +# BUILD_TYPE: Default: debug +# Maybe be one of asan|tsan|debug|release|coverage|lint +# +# Portions Copyright (c) YugaByte, Inc. + +set -euo pipefail + +# We change YB_RUN_JAVA_TEST_METHODS_SEPARATELY in a subshell in a few places and that is OK. +# shellcheck disable=SC2031 +export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 +export TSAN_OPTIONS="" +if is_mac; then + # This is needed to make sure we're using Homebrew-installed CMake on Mac OS X. + export PATH=/usr/local/bin:$PATH +fi +# gather core dumps +ulimit -c unlimited +detect_architecture +BUILD_TYPE=${BUILD_TYPE:-debug} +build_type=$BUILD_TYPE +normalize_build_type +readonly build_type +BUILD_TYPE=$build_type +readonly BUILD_TYPE +export BUILD_TYPE +export YB_USE_NINJA=1 +set_cmake_build_type_and_compiler_type +if [[ ${YB_DOWNLOAD_THIRDPARTY:-auto} == "auto" ]]; then + log "Setting YB_DOWNLOAD_THIRDPARTY=1 automatically" + export YB_DOWNLOAD_THIRDPARTY=1 +fi +log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY" +# This is normally done in set_build_root, but we need to decide earlier because this is factored +# into the decision of whether to use LTO. +decide_whether_to_use_linuxbrew +if [[ -z ${YB_LINKING_TYPE:-} ]]; then + if ! is_mac && [[ + ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ && + ${BUILD_TYPE} == "release" + ]]; then + export YB_LINKING_TYPE=full-lto + else + export YB_LINKING_TYPE=dynamic + fi + log "Automatically decided to set YB_LINKING_TYPE to ${YB_LINKING_TYPE} based on:" \ + "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}," \ + "BUILD_TYPE=${BUILD_TYPE}," \ + "YB_USE_LINUXBREW=${YB_USE_LINUXBREW}," \ + "YB_LINUXBREW_DIR=${YB_LINUXBREW_DIR:-undefined}." +else + log "YB_LINKING_TYPE is already set to ${YB_LINKING_TYPE}" +fi +log "YB_LINKING_TYPE=${YB_LINKING_TYPE}" +export YB_LINKING_TYPE \ No newline at end of file diff --git a/build-support/jenkins/test.sh b/build-support/jenkins/test.sh new file mode 100755 index 000000000000..81b1acfb4501 --- /dev/null +++ b/build-support/jenkins/test.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +# This script is invoked from the Jenkins builds to build YB and run all the unit tests. +# +# Environment variables may be used to customize operation: +# BUILD_TYPE: Default: debug +# Maybe be one of asan|tsan|debug|release|coverage|lint +# +# YB_BUILD_CPP +# Default: 1 +# Build and test C++ code if this is set to 1. +# +# YB_SKIP_BUILD +# Default: 0 +# Skip building C++ and Java code, only run tests if this is set to 1 (useful for debugging). +# This option is actually handled by yb_build.sh. +# +# YB_BUILD_JAVA +# Default: 1 +# Build and test java code if this is set to 1. +# +# DONT_DELETE_BUILD_ROOT +# Default: 0 (meaning build root will be deleted) on Jenkins, 1 (don't delete) locally. +# Skip deleting BUILD_ROOT (useful for debugging). +# +# YB_COMPILE_ONLY +# Default: 0 +# Compile the code and build a package, but don't run tests. +# +# YB_RUN_AFFECTED_TESTS_ONLY +# Default: 0 +# Try to auto-detect the set of C++ tests to run for the current set of changes relative to +# origin/master. + +# +# Portions Copyright (c) YugaByte, Inc. + +set -euo pipefail + +echo "Build script ${BASH_SOURCE[0]} is running" + +# shellcheck source=build-support/common-test-env.sh +. "${BASH_SOURCE%/*}/../common-test-env.sh" + +# ------------------------------------------------------------------------------------------------- +# Functions + +cleanup() { + if [[ -n ${BUILD_ROOT:-} && ${DONT_DELETE_BUILD_ROOT} == "0" ]]; then + log "Running the script to clean up build artifacts..." + "$YB_BUILD_SUPPORT_DIR/jenkins/post-build-clean.sh" + fi +} + +# ================================================================================================= +# Main script +# ================================================================================================= + +log "Running with Bash version $BASH_VERSION" + +cd "$YB_SRC_ROOT" +if ! "$YB_BUILD_SUPPORT_DIR/common-build-env-test.sh"; then + fatal "Test of the common build environment failed, cannot proceed." +fi + +activate_virtualenv +set_pythonpath + +# shellcheck source=build-support/jenkins/common-lto.sh +. "${BASH_SOURCE%/*}/common-lto.sh" + +# ------------------------------------------------------------------------------------------------- +# Build root setup and build directory cleanup +# ------------------------------------------------------------------------------------------------- + +# shellcheck disable=SC2119 +set_build_root + +set_common_test_paths + +# As soon as we know build root, we need to do the necessary workspace cleanup. +if is_jenkins; then + # Delete the build root by default on Jenkins. + DONT_DELETE_BUILD_ROOT=${DONT_DELETE_BUILD_ROOT:-0} +else + log "Not running on Jenkins, not deleting the build root by default." + # Don't delete the build root by default. + DONT_DELETE_BUILD_ROOT=${DONT_DELETE_BUILD_ROOT:-1} +fi + +export BUILD_ROOT + +# ------------------------------------------------------------------------------------------------- +# End of build root setup and build directory cleanup +# ------------------------------------------------------------------------------------------------- + +# We need to set this prior to the first invocation of yb_build.sh. +export YB_SKIP_FINAL_LTO_LINK=1 + +find_or_download_ysql_snapshots +find_or_download_thirdparty +validate_thirdparty_dir +detect_toolchain +log_thirdparty_and_toolchain_details +find_make_or_ninja_and_update_cmake_opts + +log "YB_USE_NINJA=$YB_USE_NINJA" +log "YB_NINJA_PATH=${YB_NINJA_PATH:-undefined}" + +set_java_home + +export YB_DISABLE_LATEST_SYMLINK=1 +remove_latest_symlink + +if is_jenkins; then + log "Running on Jenkins, will re-create the Python virtualenv" + # YB_RECREATE_VIRTUALENV is used in common-build-env.sh. + # shellcheck disable=SC2034 + YB_RECREATE_VIRTUALENV=1 +fi + +log "Running with PATH: ${PATH}" + +set +e +for python_command in python python2 python2.7 python3; do + log "Location of $python_command: $( which "$python_command" )" +done +set -e + +log "Running Python tests" +time run_python_tests +log "Finished running Python tests (see timing information above)" + +log "Running a light-weight lint script on our Java code" +time lint_java_code +log "Finished running a light-weight lint script on the Java code" + +# TODO: deduplicate this with similar logic in yb-jenkins-build.sh. +YB_BUILD_JAVA=${YB_BUILD_JAVA:-1} +YB_BUILD_CPP=${YB_BUILD_CPP:-1} + +if [[ -z ${YB_RUN_AFFECTED_TESTS_ONLY:-} ]] && is_jenkins_phabricator_build; then + log "YB_RUN_AFFECTED_TESTS_ONLY is not set, and this is a Jenkins Phabricator test." \ + "Setting YB_RUN_AFFECTED_TESTS_ONLY=1 automatically." + export YB_RUN_AFFECTED_TESTS_ONLY=1 +fi +export YB_RUN_AFFECTED_TESTS_ONLY=${YB_RUN_AFFECTED_TESTS_ONLY:-0} +log "YB_RUN_AFFECTED_TESTS_ONLY=${YB_RUN_AFFECTED_TESTS_ONLY}" + +export YB_SKIP_BUILD=${YB_SKIP_BUILD:-0} +if [[ ${YB_SKIP_BUILD} == "1" ]]; then + export NO_REBUILD_THIRDPARTY=1 +fi + +YB_SKIP_CPP_COMPILATION=${YB_SKIP_CPP_COMPILATION:-0} +YB_COMPILE_ONLY=${YB_COMPILE_ONLY:-0} + +CTEST_OUTPUT_PATH="${BUILD_ROOT}"/ctest.log +CTEST_FULL_OUTPUT_PATH="${BUILD_ROOT}"/ctest-full.log + +TEST_LOG_DIR="${BUILD_ROOT}/test-logs" + +# If we're running inside Jenkins (the BUILD_ID is set), then install an exit handler which will +# clean up all of our build results. +if is_jenkins; then + trap cleanup EXIT +fi + +configure_remote_compilation + +export NO_REBUILD_THIRDPARTY=1 + +THIRDPARTY_BIN=$YB_SRC_ROOT/thirdparty/installed/bin +export PPROF_PATH=$THIRDPARTY_BIN/pprof + +# Configure the build +# + +cd "$BUILD_ROOT" + +if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then + ( + set -x + # Remove the compilation command file, even if we have not deleted the build root. + rm -f "$BUILD_ROOT/compile_commands.json" + ) +fi + +# Only enable test core dumps for certain build types. +if [[ ${BUILD_TYPE} != "asan" ]]; then + # TODO: actually make this take effect. The issue is that we might not be able to set ulimit + # unless the OS configuration enables us to. + export YB_TEST_ULIMIT_CORE=unlimited +fi + +detect_num_cpus + +declare -i EXIT_STATUS=0 + +set +e +if [[ -d /tmp/yb-port-locks ]]; then + # Allow other users to also run minicluster tests on this machine. + chmod a+rwx /tmp/yb-port-locks +fi +set -e + +FAILURES="" + +if [[ ${YB_BUILD_CPP} == "1" ]] && ! which ctest >/dev/null; then + fatal "ctest not found, won't be able to run C++ tests" +fi + +export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=1 + +# ------------------------------------------------------------------------------------------------- +# Running initdb +# ------------------------------------------------------------------------------------------------- + +export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=0 + +# ------------------------------------------------------------------------------------------------- +# Dependency graph analysis allowing to determine what tests to run. +# ------------------------------------------------------------------------------------------------- + +if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then + if ! ( set -x + "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + --build-root "${BUILD_ROOT}" \ + self-test \ + --rebuild-graph ); then + # Trying to diagnose this error: + # https://gist.githubusercontent.com/mbautin/c5c6f14714f7655c10620d8e658e1f5b/raw + log "dependency_graph.py failed, listing all pb.{h,cc} files in the build directory" + ( set -x; find "$BUILD_ROOT" -name "*.pb.h" -or -name "*.pb.cc" ) + fatal "Dependency graph construction failed" + fi +fi + +# Save the current HEAD commit in case we build Java below and add a new commit. This is used for +# the following purposes: +# - So we can upload the release under the correct commit, from Jenkins, to then be picked up from +# itest, from the snapshots bucket. +# - For picking up the changeset corresponding the the current diff being tested and detecting what +# tests to run in Phabricator builds. If we just diff with origin/master, we'll always pick up +# pom.xml changes we've just made, forcing us to always run Java tests. +current_git_commit=$(git rev-parse HEAD) + +export YB_MVN_LOCAL_REPO=$BUILD_ROOT/m2_repository + +# ------------------------------------------------------------------------------------------------- +# Run tests, either on Spark or locally. +# If YB_COMPILE_ONLY is set to 1, we skip running all tests (Java and C++). + +set_sanitizer_runtime_options + +# To reduce Jenkins archive size, let's gzip Java logs and delete per-test-method logs in case +# of no test failures. +export YB_GZIP_PER_TEST_METHOD_LOGS=1 +export YB_GZIP_TEST_LOGS=1 +export YB_DELETE_SUCCESSFUL_PER_TEST_METHOD_LOGS=1 + +if [[ ${YB_COMPILE_ONLY} != "1" ]]; then + if spark_available; then + if [[ ${YB_BUILD_CPP} == "1" || ${YB_BUILD_JAVA} == "1" ]]; then + log "Will run tests on Spark" + run_tests_extra_args=() + if [[ ${YB_BUILD_JAVA} == "1" ]]; then + run_tests_extra_args+=( "--java" ) + fi + if [[ ${YB_BUILD_CPP} == "1" ]]; then + run_tests_extra_args+=( "--cpp" ) + fi + if [[ ${YB_RUN_AFFECTED_TESTS_ONLY} == "1" ]]; then + test_conf_path="${BUILD_ROOT}/test_conf.json" + # YB_GIT_COMMIT_FOR_DETECTING_TESTS allows overriding the commit to use to detect the set + # of tests to run. Useful when testing this script. + ( + set -x + "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + --build-root "${BUILD_ROOT}" \ + --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \ + --output-test-config "${test_conf_path}" \ + affected + ) + run_tests_extra_args+=( "--test_conf" "${test_conf_path}" ) + unset test_conf_path + fi + if is_linux || (is_mac && ! is_src_root_on_nfs); then + log "Will create an archive for Spark workers with all the code instead of using NFS." + run_tests_extra_args+=( "--send_archive_to_workers" ) + fi + # Workers use /private path, which caused mis-match when check is done by yb_dist_tests that + # YB_MVN_LOCAL_REPO is in source tree. So unsetting value here to allow default. + if is_mac; then + unset YB_MVN_LOCAL_REPO + fi + + NUM_REPETITIONS="${YB_NUM_REPETITIONS:-1}" + log "NUM_REPETITIONS is set to ${NUM_REPETITIONS}" + if [[ ${NUM_REPETITIONS} -gt 1 ]]; then + log "Repeating each test ${NUM_REPETITIONS} times" + run_tests_extra_args+=( "--num_repetitions" "${NUM_REPETITIONS}" ) + fi + + set +u # because extra_args can be empty + if ! run_tests_on_spark "${run_tests_extra_args[@]}"; then + set -u + EXIT_STATUS=1 + FAILURES+=$'Distributed tests on Spark (C++ and/or Java) failed\n' + log "Some tests that were run on Spark failed" + fi + set -u + unset extra_args + else + log "Neither C++ or Java tests are enabled, nothing to run on Spark." + fi + else + # A single-node way of running tests (without Spark). + + if [[ ${YB_BUILD_CPP} == "1" ]]; then + log "Run C++ tests in a non-distributed way" + export GTEST_OUTPUT="xml:${TEST_LOG_DIR}/" # Enable JUnit-compatible XML output. + + if ! spark_available; then + log "Did not find Spark on the system, falling back to a ctest-based way of running tests" + set +e + # We don't double-quote EXTRA_TEST_FLAGS on purpose, to allow specifying multiple flags. + # shellcheck disable=SC2086 + time ctest "-j${NUM_PARALLEL_TESTS}" ${EXTRA_TEST_FLAGS:-} \ + --output-log "${CTEST_FULL_OUTPUT_PATH}" \ + --output-on-failure 2>&1 | tee "${CTEST_OUTPUT_PATH}" + ctest_exit_code=$? + set -e + if [[ $ctest_exit_code -ne 0 ]]; then + EXIT_STATUS=1 + FAILURES+=$'C++ tests failed with exit code ${ctest_exit_code}\n' + fi + fi + log "Finished running C++ tests (see timing information above)" + fi + + if [[ ${YB_BUILD_JAVA} == "1" ]]; then + set_test_invocation_id + log "Running Java tests in a non-distributed way" + if ! time run_all_java_test_methods_separately; then + EXIT_STATUS=1 + FAILURES+=$'Java tests failed\n' + fi + log "Finished running Java tests (see timing information above)" + # shellcheck disable=SC2119 + kill_stuck_processes + fi + fi +fi + +# Finished running tests. +remove_latest_symlink + +log "Aggregating test reports" +"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ + --yb-src-root "${YB_SRC_ROOT}" \ + --output-dir "${YB_SRC_ROOT}" \ + --build-type "${build_type}" \ + --compiler-type "${YB_COMPILER_TYPE}" \ + --build-root "${BUILD_ROOT}" + +if [[ -n ${FAILURES} ]]; then + heading "Failure summary" + echo >&2 "${FAILURES}" +fi + +exit ${EXIT_STATUS} \ No newline at end of file diff --git a/build-support/jenkins/yb-jenkins-build.sh b/build-support/jenkins/yb-jenkins-build.sh index 25b211c0b334..03656e2d13d0 100755 --- a/build-support/jenkins/yb-jenkins-build.sh +++ b/build-support/jenkins/yb-jenkins-build.sh @@ -34,8 +34,6 @@ Environment variables: BUILD_TYPE Passed directly to build-and-test.sh. The default value is determined based on the job name if this environment variable is not specified or if the value is "auto". - YB_NUM_TESTS_TO_RUN - Maximum number of tests ctest should run before exiting. Used for testing Jenkins scripts. EOT } @@ -77,15 +75,6 @@ for branch_name in $( git for-each-ref --format="%(refname)" refs/heads/ ); do fi done -if [ -n "${YB_NUM_TESTS_TO_RUN:-}" ]; then - - if [[ ! "$YB_NUM_TESTS_TO_RUN" =~ ^[0-9]+$ ]]; then - echo "Invalid number of tests to run: $YB_NUM_TESTS_TO_RUN" >&2 - exit 1 - fi - export EXTRA_TEST_FLAGS="-I1,$YB_NUM_TESTS_TO_RUN" -fi - export YB_MINIMIZE_VERSION_DEFINES_CHANGES=1 export YB_MINIMIZE_RECOMPILATION=1 @@ -118,7 +107,7 @@ if is_mac; then fi set +e -"$YB_BUILD_SUPPORT_DIR"/jenkins/build-and-test.sh +"$YB_BUILD_SUPPORT_DIR"/jenkins/build.sh exit_code=$? set -e diff --git a/build-support/jenkins/yb-jenkins-test.sh b/build-support/jenkins/yb-jenkins-test.sh new file mode 100755 index 000000000000..50eeebb61e95 --- /dev/null +++ b/build-support/jenkins/yb-jenkins-test.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# +# Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail + +# shellcheck source=build-support/common-test-env.sh +. "${0%/*}/../common-test-env.sh" + +print_help() { + cat <<-EOT +Usage: ${0##*} +Options: + -h, --help + Show help + --delete-arc-patch-branches + Delete branches starting with "arcpatch-D..." (except the current branch) so that the Jenkins + Phabricator plugin does not give up after three attempts. + +Environment variables: + JOB_NAME + Jenkins job name. + BUILD_TYPE + Passed directly to build-and-test.sh. The default value is determined based on the job name + if this environment variable is not specified or if the value is "auto". + YB_NUM_TESTS_TO_RUN + Maximum number of tests ctest should run before exiting. Used for testing Jenkins scripts. +EOT +} + +echo "Build script ${BASH_SOURCE[0]} is running" + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + print_help + exit 0 + ;; + *) + echo "Invalid option: $1" >&2 + exit 1 + esac + shift +done + +JOB_NAME=${JOB_NAME:-} +build_type=${BUILD_TYPE:-} +set_build_type_based_on_jenkins_job_name +readonly BUILD_TYPE=$build_type +export BUILD_TYPE + +echo "Build type: ${BUILD_TYPE}"; + +set_compiler_type_based_on_jenkins_job_name + +if [ -n "${YB_NUM_TESTS_TO_RUN:-}" ]; then + + if [[ ! "$YB_NUM_TESTS_TO_RUN" =~ ^[0-9]+$ ]]; then + echo "Invalid number of tests to run: $YB_NUM_TESTS_TO_RUN" >&2 + exit 1 + fi + export EXTRA_TEST_FLAGS="-I1,$YB_NUM_TESTS_TO_RUN" +fi + +export YB_MINIMIZE_VERSION_DEFINES_CHANGES=1 +export YB_MINIMIZE_RECOMPILATION=1 + +export YB_BUILD_JAVA=${YB_BUILD_JAVA:-1} +export YB_BUILD_PYTHON=${YB_BUILD_PYTHON:-0} +export YB_BUILD_CPP=${YB_BUILD_CPP:-1} + +set +e +"$YB_BUILD_SUPPORT_DIR"/jenkins/test.sh +exit_code=$? + +set -e + +exit $exit_code diff --git a/docs/assets/scss/_yb_image.scss b/docs/assets/scss/_yb_image.scss index 911cf603b147..2de6c94b056f 100644 --- a/docs/assets/scss/_yb_image.scss +++ b/docs/assets/scss/_yb_image.scss @@ -40,3 +40,107 @@ } } } + +body { + &.image-popped-up { + overflow: hidden; + } +} + +p > img, +.td-content > img:not(.icon) { + cursor: pointer; +} + +.image-popup { + position: fixed; + inset: 0; + z-index: -1; + opacity: 0; + transition: opacity 500ms ease; + align-items: center; + justify-content: center; + display: flex; + + &.open { + opacity: 1; + z-index: 999999; + } + + .img-scroll { + width: auto; + background: #fff; + border-radius: 5px; + max-width: 90vw; + max-height: 90vh; + overflow: visible; + box-sizing: border-box; + position: relative; + margin-top: 5vh; + + @media (max-width: 767px) { + max-height: 80vh; + margin-top: 10vh; + } + + img { + width: auto; + height: auto; + max-width: 90vw; + max-height: 90vh; + + &[src*="svg"] { + width: 80vw; + } + } + + i { + position: absolute; + width: 25px; + height: 25px; + top: -25px; + right: -35px; + border-radius: 50%; + background: #fff; + z-index: 6; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + transition: all 250ms ease; + + @media (max-width: 767px) { + top: -35px; + right: -15px; + } + + &:hover { + background: #f1f1f1; + } + + &::before, + &::after { + content: ""; + width: 16px; + height: 1px; + background: #000; + position: absolute; + top: 12px; + left: 5px; + transform: rotate(45deg); + } + + &::after { + transform: rotate(-45deg); + } + } + } + + i.bg-drop { + position: absolute; + inset: 0; + background: rgba(0, 0, 0, 0.7); + backdrop-filter: blur(2px); + cursor: pointer; + } +} diff --git a/docs/content/preview/admin/yb-admin.md b/docs/content/preview/admin/yb-admin.md index e42fe24a8277..6fde2e25644c 100644 --- a/docs/content/preview/admin/yb-admin.md +++ b/docs/content/preview/admin/yb-admin.md @@ -931,7 +931,7 @@ yb-admin create_snapshot_schedule \ * *retention-time*: The number of minutes to keep a snapshot before deleting it. * *filter-expression*: The set of objects to include in the snapshot. -The filter expression is a list of acceptable objects, which can be either raw tables, or keyspaces (YCQL) or databases (YSQL). For proper consistency guarantees, **Yugabyte recommends setting this up on a per-keyspace (YCQL) or per-database (YSQL) level**. +The filter expression is a list of acceptable objects, which can be either raw tables, or keyspaces (YCQL) or databases (YSQL). For proper consistency guarantees, **it is recommended to set this up on a per-keyspace (YCQL) or per-database (YSQL) level**. **Example** diff --git a/docs/content/preview/api/ycql/batch.md b/docs/content/preview/api/ycql/batch.md index f34325df22fd..2b2bea616287 100644 --- a/docs/content/preview/api/ycql/batch.md +++ b/docs/content/preview/api/ycql/batch.md @@ -161,7 +161,7 @@ cqlsh:sample> SELECT * FROM test; (1 rows) ``` -{{< note Type="Note" >}} +{{< note title="Note" >}} `BEGIN/END TRANSACTION` doesn't currently support `RETURNS STATUS AS ROW`. diff --git a/docs/content/preview/api/ycql/dml_transaction.md b/docs/content/preview/api/ycql/dml_transaction.md index 7afa9984f704..cdcc0805fb39 100644 --- a/docs/content/preview/api/ycql/dml_transaction.md +++ b/docs/content/preview/api/ycql/dml_transaction.md @@ -145,7 +145,7 @@ ycqlsh:example> SELECT account_name, account_type, balance, writetime(balance) F Smith | savings | 2000 | 1523313964363056 ``` -{{< note Type="Note" >}} +{{< note title="Note" >}} `BEGIN/END TRANSACTION` doesn't currently support `RETURNS STATUS AS ROW`. {{< /note >}} diff --git a/docs/content/preview/api/ysql/the-sql-language/statements/cmd_reset.md b/docs/content/preview/api/ysql/the-sql-language/statements/cmd_reset.md index 2cdbc6eccfea..8e21f9f1f0ad 100644 --- a/docs/content/preview/api/ysql/the-sql-language/statements/cmd_reset.md +++ b/docs/content/preview/api/ysql/the-sql-language/statements/cmd_reset.md @@ -44,11 +44,11 @@ Use the `RESET` statement to restore the value of a run-time parameter to the de ## Semantics -{{< note Type="Note" >}} +{{< note title="Note" >}} Although the values of a parameter can be set, displayed, and reset, the effect of these parameters are not yet supported in Yugabyte. The factory settings or default behaviors will be used for the moment. -{{ / }} +{{< /note >}} ### *configuration_parameter* diff --git a/docs/content/preview/contribute/docs/all-page-elements.md b/docs/content/preview/contribute/docs/all-page-elements.md new file mode 100644 index 000000000000..667747449c6f --- /dev/null +++ b/docs/content/preview/contribute/docs/all-page-elements.md @@ -0,0 +1,339 @@ +--- +title: YugabyteDB Managed quick start +headerTitle: Page with elements +linkTitle: Page with elements +headcontent: Just about every page element on a single page +description: YugabyteDB Documentation page elements on a single page. +layout: single +type: docs +menu: + preview: + identifier: all-page-elements + parent: docs-edit + weight: 9000 +--- + +This page demonstrates styles and widgets used for the YugabyteDB Documentation site. + +## Tab widgets + + + +This is a top-level tab widget, that uses different files for each tab. Everything that follows is in this tab's file. If you change tabs, you get a whole new page. + +### Second tab widget style + + + +This is a second-level tab widget, that uses different files for each tab - same as the one above, just styled differently. + +Everything that follows is in this tab's file. If you change tabs, you get a whole new page. + +### In-page tab widget + +This tab widget doesn't use separate files to fill in the content and then link between. Here the content is placed inside the `tabpane` shortcode. + +{{< tabpane text=true >}} + + {{% tab header="Tab" lang="java" %}} + +Another tab style; here all the tab contents are in-page. + +### Headings inside this widget don't show up in RightNav + +To build and run the application, do the following: + +1. Clone the sample application to your computer: + + ```sh + git clone https://github.com/YugabyteDB-Samples/yugabyte-simple-java-app.git && cd yugabyte-simple-java-app + ``` + +1. Provide connection parameters. + + The application needs to establish a connection to the YugabyteDB cluster. To do this: + + - Set the following configuration parameters: + + - **host** - the host name of your YugabyteDB cluster. To obtain a YugabyteDB Managed cluster host name, sign in to YugabyteDB Managed, select your cluster on the **Clusters** page, and click **Settings**. The host is displayed under **Connection Parameters**. + - **port** - the port number that will be used by the JDBC driver (the default YugabyteDB YSQL port is 5433). + + - Save the file. + +The end of this tab. + + {{% /tab %}} + + {{% tab header="Another tab" lang="go" %}} + +The contents of the next tab. You can keep adding tabs in similar fashion. + +### Headings inside this widget don't show up in RightNav + +To build and run the application, do the following: + +1. Clone the sample application to your computer: + + ```sh + git clone https://github.com/YugabyteDB-Samples/yugabyte-simple-go-app.git && cd yugabyte-simple-go-app + ``` + +1. Provide connection parameters. + + {{% /tab %}} + +{{< /tabpane >}} + +### Pills + +These pills automatically re-flow based on page width; just keep adding list items. Use these pills for landing pages or as an alternative to tabs. + + + +## Table + +The following is a basic markdown table. + +| Table | A column | +| :---- | :------- | +| A row | Another column in a table. Maybe to describe stuff. Might have bulleted lists etc, but that all has to be done using HTML. | +| Another row | Another row in a table. | +| Another row | Another column in a table. | + +## Notes and blockquote + +An ordinary paragraph. + +>**Blockquote** +> +>Blockquote text +> +>Some more blockquote text. + +{{< note title="Note" >}} + +This is an actual Note. Pay special attention to this thing, maybe. + +{{< /note >}} + +{{< tip title="Tip" >}} + +This is an actual Tip. Maybe there's a cool alternative to what you are doing. + +{{< /tip >}} + +{{< warning title="Warning" >}} + +This is an actual Warning. Whatever you are doing may threaten your data if you do it wrong. + +{{< /warning >}} + +## Images and video + +What follows is an image: + +![Connect using cloud shell](/images/yb-cloud/cloud-connect-shell.gif) + +What follows is an embedded YouTube video: + +{{< youtube id="qYMcNzWotkI" title="Deploy a fault tolerant cluster in YugabyteDB Managed" >}} + +## Lists + +What follows are some numbered steps, some code, text indented under a step: + +1. Numbered list with indented lists. + + 1. Second level numbered list indent. + + 1. Second level numbered list indent. + + 1. Third level numbered list indent. + + 1. Third level numbered list indent. + +1. Second list item. Some text in **bold**. Some text in _italics_. + +1. Third list item. + + Indented text under a list. + + ```output + Some code + ``` + +1. Fourth list item. + +Bulleted lists with levels of indent: + +- Bulleted list with indented lists. + + - Second level Bulleted list indent. + + - Second level Bulleted list indent. + + - Third level Bulleted list indent. + + - Third level Bulleted list indent. + +### Heading 3 + +#### Heading 4 + +Some bullets: + +- a cluster deployed in YugabyteDB Managed. +- the cluster CA certificate; YugabyteDB Managed uses TLS to secure connections to the database. +- your computer added to the cluster IP allow list. + +Refer to [Before you begin](../../../develop/build-apps/cloud-add-ip/). + +## Glossary entries + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. +: Another paragraph in the definition. + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. +: Another paragraph in the definition. + +## Details tag + +The details HTML tag is used to create an interactive widget that the user can open and close. By default, the widget is closed. When open, it expands, and displays the contents. + +
+ Long code block + +These contents might be a very long bit of code or somesuch. + +
+ +## Syntax documentation + +```sh +ysqlsh [