From 66af1d42e02c5e12b4df0c738b5123dad6226076 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Mon, 19 Dec 2022 14:01:53 -0800 Subject: [PATCH] build: add samples test with credentials (#3708) * build: add samples test with credentials Co-authored-by: Owl Bot --- ci/cloudbuild_with_credentials.yaml | 48 +++++++++++++++++++ ...les-presubmit-node14-with-credentials.yaml | 17 +++++++ ci/export/samples-presubmit-node14.yaml | 2 +- ci/export_triggers.sh | 4 +- ci/import_triggers.sh | 2 + ci/run_conditional_tests.sh | 27 +++++++++-- 6 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 ci/cloudbuild_with_credentials.yaml create mode 100644 ci/export/samples-presubmit-node14-with-credentials.yaml diff --git a/ci/cloudbuild_with_credentials.yaml b/ci/cloudbuild_with_credentials.yaml new file mode 100644 index 00000000000..d49a9d8c3d0 --- /dev/null +++ b/ci/cloudbuild_with_credentials.yaml @@ -0,0 +1,48 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +options: + dynamic_substitutions: true + +substitutions: + _BUILD_TYPE: "presubmit" + _TEST_TYPE: "system" + _NODE_VERSION: "14" + +logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node/${_BUILD_TYPE}/${COMMIT_SHA}/${TRIGGER_NAME}' +timeout: 7200s + +steps: + - name: 'gcr.io/kaniko-project/executor:v1.9.1' + args: [ + '--log-format=text', + '--context=dir:///workspace/testing', + '--build-arg=NODE_VERSION=${_NODE_VERSION}', + '--dockerfile=ci/Dockerfile', + '--cache=true', + '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-${_NODE_VERSION}', + '--push-retry=3', + '--image-fs-extract-retry=3' + ] + - name: gcr.io/cloud-builders/gcloud + entrypoint: 'bash' + args: [ '-c', "gcloud secrets versions access latest --project=cloud-devrel-kokoro-resources --secret=long-door-651-kokoro-system-test-service-account --format='get(payload.data)' | tr '_-' '/+' | base64 -d > /workspace/google_application_credentials.json" ] + - name: gcr.io/${PROJECT_ID}/google-cloud-node-${_NODE_VERSION} + id: "run-tests" + timeout: 7200s + entrypoint: "ci/run_conditional_tests.sh" + env: + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TEST_TYPE=${_TEST_TYPE}' + - 'GOOGLE_APPLICATION_CREDENTIALS=/workspace/google_application_credentials.json' diff --git a/ci/export/samples-presubmit-node14-with-credentials.yaml b/ci/export/samples-presubmit-node14-with-credentials.yaml new file mode 100644 index 00000000000..878e5b50bb1 --- /dev/null +++ b/ci/export/samples-presubmit-node14-with-credentials.yaml @@ -0,0 +1,17 @@ +createTime: '2022-08-05T20:10:51.866623226Z' +description: Presubmit build with node 14 +filename: ci/cloudbuild_with_credentials.yaml +github: + name: google-cloud-node + owner: googleapis + pullRequest: + branch: ^main$ + commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY +id: 209625ae-8ae4-4742-a175-4c1330cea557 +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +name: samples-presubmit-node14-with-credentials +serviceAccount: projects/long-door-651/serviceAccounts/kokoro-system-test@long-door-651.iam.gserviceaccount.com +substitutions: + _BUILD_TYPE: presubmit + _LOGS_BUCKET: cloud-node-public-logs + _TEST_TYPE: samples diff --git a/ci/export/samples-presubmit-node14.yaml b/ci/export/samples-presubmit-node14.yaml index dbb66ec45b3..ae2dcfc0ec7 100644 --- a/ci/export/samples-presubmit-node14.yaml +++ b/ci/export/samples-presubmit-node14.yaml @@ -14,4 +14,4 @@ serviceAccount: projects/long-door-651/serviceAccounts/kokoro-system-test@long-d substitutions: _BUILD_TYPE: presubmit _LOGS_BUCKET: cloud-node-public-logs - _TEST_TYPE: samples + _TEST_TYPE: samples \ No newline at end of file diff --git a/ci/export_triggers.sh b/ci/export_triggers.sh index 78ab45c7d81..2c427302b5f 100755 --- a/ci/export_triggers.sh +++ b/ci/export_triggers.sh @@ -44,4 +44,6 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do gcloud beta builds triggers export "samples-continuous-node${NODE_VERSION}" --destination "ci/export/samples-continuous-node${NODE_VERSION}.yaml" echo "exporting nightly build for node${NODE_VERSION}" gcloud beta builds triggers export "samples-nightly-node${NODE_VERSION}" --destination "ci/export/samples-nightly-node${NODE_VERSION}.yaml" -done \ No newline at end of file + echo "exporting presubmit build for node${NODE_VERSION} with credentials" + gcloud beta builds triggers export "samples-presubmit-node${NODE_VERSION}-with-credentials" --destination "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml" +done diff --git a/ci/import_triggers.sh b/ci/import_triggers.sh index 2f0ddc871de..a7c8c1e66fc 100755 --- a/ci/import_triggers.sh +++ b/ci/import_triggers.sh @@ -44,4 +44,6 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do gcloud beta builds triggers import --source "ci/export/samples-continuous-node${NODE_VERSION}.yaml" echo "importing nightly build for node${NODE_VERSION}" gcloud beta builds triggers import --source "ci/export/samples-nightly-node${NODE_VERSION}.yaml" + echo "importing presubmit build for node${NODE_VERSION} with credentials" + gcloud beta builds triggers import --source "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml" done \ No newline at end of file diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh index 03cc59a6afb..421a3c9033b 100755 --- a/ci/run_conditional_tests.sh +++ b/ci/run_conditional_tests.sh @@ -24,6 +24,7 @@ export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..) # A script file for running the test in a sub project. test_script="${PROJECT_ROOT}/ci/run_single_test.sh" + if [ ${BUILD_TYPE} == "presubmit" ]; then # For presubmit build, we want to know the difference from the # common commit in origin/main. @@ -67,6 +68,9 @@ subdirs=( ) RETVAL=0 +# These following APIs need an explicit credential file to run properly (or oAuth2, which we don't support in this repo). +# When we hit these packages, we will run the "samples with credentials" trigger, which contains the credentials as an env variable +tests_with_credentials="packages/google-analytics-admin/ packages/google-area120-tables/ packages/google-analytics-data/ packages/google-iam-credentials/" for subdir in ${subdirs[@]}; do for d in `ls -d ${subdir}/*/`; do @@ -80,12 +84,29 @@ for subdir in ${subdirs[@]}; do if [[ "${changed}" -eq 0 ]]; then echo "no change detected in ${d}, skipping" else - echo "change detected in ${d}" - should_test=true + if [[ "${TEST_TYPE}" == "system" ]]; then + echo "change detected in ${d} for system test" + should_test=true + elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + echo "change detected in ${d} in a directory that needs credentials" + should_test=true + elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + echo "change detected in ${d}" + should_test=true + fi fi else # If GIT_DIFF_ARG is empty, run all the tests. - should_test=true + if [[ "${TEST_TYPE}" == "system" ]]; then + echo "run system test for ${d}" + should_test=true + elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + echo "run tests with credentials in ${d}" + should_test=true + elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + echo "run tests in ${d}" + should_test=true + fi fi if [ "${should_test}" = true ]; then echo "running test in ${d}"