build: add samples test with credentials (#3708)

* build: add samples test with credentials
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

ci/cloudbuild_with_credentials.yaml

@@ -0,0 +1,48 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+options:
+  dynamic_substitutions: true
+
+substitutions:
+  _BUILD_TYPE: "presubmit"
+  _TEST_TYPE: "system"
+  _NODE_VERSION: "14"
+
+logsBucket: 'gs://${_LOGS_BUCKET}/logs/google-cloud-node/${_BUILD_TYPE}/${COMMIT_SHA}/${TRIGGER_NAME}'
+timeout: 7200s
+
+steps:
+  - name: 'gcr.io/kaniko-project/executor:v1.9.1'
+    args: [
+      '--log-format=text',
+      '--context=dir:///workspace/testing',
+      '--build-arg=NODE_VERSION=${_NODE_VERSION}',
+      '--dockerfile=ci/Dockerfile',
+      '--cache=true',
+      '--destination=gcr.io/${PROJECT_ID}/google-cloud-node-${_NODE_VERSION}',
+      '--push-retry=3',
+      '--image-fs-extract-retry=3'
+    ]
+  - name: gcr.io/cloud-builders/gcloud
+    entrypoint: 'bash'
+    args: [ '-c', "gcloud secrets versions access latest --project=cloud-devrel-kokoro-resources --secret=long-door-651-kokoro-system-test-service-account --format='get(payload.data)'  | tr '_-' '/+' | base64 -d > /workspace/google_application_credentials.json" ]
+  - name: gcr.io/${PROJECT_ID}/google-cloud-node-${_NODE_VERSION}
+    id: "run-tests"
+    timeout: 7200s
+    entrypoint: "ci/run_conditional_tests.sh"
+    env:
+    - 'BUILD_TYPE=${_BUILD_TYPE}'
+    - 'TEST_TYPE=${_TEST_TYPE}'
+    - 'GOOGLE_APPLICATION_CREDENTIALS=/workspace/google_application_credentials.json'

ci/export/samples-presubmit-node14-with-credentials.yaml

@@ -0,0 +1,17 @@
+createTime: '2022-08-05T20:10:51.866623226Z'
+description: Presubmit build with node 14
+filename: ci/cloudbuild_with_credentials.yaml
+github:
+  name: google-cloud-node
+  owner: googleapis
+  pullRequest:
+    branch: ^main$
+    commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY
+id: 209625ae-8ae4-4742-a175-4c1330cea557
+includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS
+name: samples-presubmit-node14-with-credentials
+serviceAccount: projects/long-door-651/serviceAccounts/[email protected]
+substitutions:
+  _BUILD_TYPE: presubmit
+  _LOGS_BUCKET: cloud-node-public-logs
+  _TEST_TYPE: samples

ci/export/samples-presubmit-node14.yaml

@@ -14,4 +14,4 @@ serviceAccount: projects/long-door-651/serviceAccounts/kokoro-system-test@long-d
 substitutions:
   _BUILD_TYPE: presubmit
   _LOGS_BUCKET: cloud-node-public-logs
-  _TEST_TYPE: samples
+  _TEST_TYPE: samples

ci/export_triggers.sh

@@ -44,4 +44,6 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do
     gcloud beta builds triggers export "samples-continuous-node${NODE_VERSION}" --destination "ci/export/samples-continuous-node${NODE_VERSION}.yaml"
     echo "exporting nightly build for node${NODE_VERSION}"
     gcloud beta builds triggers export "samples-nightly-node${NODE_VERSION}" --destination "ci/export/samples-nightly-node${NODE_VERSION}.yaml"
-done
+    echo "exporting presubmit build for node${NODE_VERSION} with credentials"
+    gcloud beta builds triggers export "samples-presubmit-node${NODE_VERSION}-with-credentials" --destination "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml"
+done

ci/import_triggers.sh

@@ -44,4 +44,6 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do
     gcloud beta builds triggers import --source "ci/export/samples-continuous-node${NODE_VERSION}.yaml"
     echo "importing nightly build for node${NODE_VERSION}"
     gcloud beta builds triggers import --source "ci/export/samples-nightly-node${NODE_VERSION}.yaml"
+    echo "importing presubmit build for node${NODE_VERSION} with credentials"
+    gcloud beta builds triggers import --source "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml"
 done

ci/run_conditional_tests.sh

@@ -24,6 +24,7 @@ export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
 # A script file for running the test in a sub project.
 test_script="${PROJECT_ROOT}/ci/run_single_test.sh"
 
+
 if [ ${BUILD_TYPE} == "presubmit" ]; then
     # For presubmit build, we want to know the difference from the
     # common commit in origin/main.
@@ -67,6 +68,9 @@ subdirs=(
 )
 
 RETVAL=0
+# These following APIs need an explicit credential file to run properly (or oAuth2, which we don't support in this repo). 
+# When we hit these packages, we will run the "samples with credentials" trigger, which contains the credentials as an env variable
+tests_with_credentials="packages/google-analytics-admin/ packages/google-area120-tables/ packages/google-analytics-data/ packages/google-iam-credentials/"
 
 for subdir in ${subdirs[@]}; do
     for d in `ls -d ${subdir}/*/`; do
@@ -80,12 +84,29 @@ for subdir in ${subdirs[@]}; do
             if [[ "${changed}" -eq 0 ]]; then
                 echo "no change detected in ${d}, skipping"
             else
-                echo "change detected in ${d}"
-                should_test=true
+                if [[ "${TEST_TYPE}" == "system" ]]; then
+                    echo "change detected in ${d} for system test"
+                    should_test=true
+                elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+                    echo "change detected in ${d} in a directory that needs credentials"
+                    should_test=true
+                elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+                    echo "change detected in ${d}"
+                    should_test=true
+                fi
             fi
         else
             # If GIT_DIFF_ARG is empty, run all the tests.
-            should_test=true
+            if [[ "${TEST_TYPE}" == "system" ]]; then
+                echo "run system test for ${d}"
+                should_test=true
+            elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+                echo "run tests with credentials in ${d}"
+                should_test=true
+            elif ! [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+                echo "run tests in ${d}"
+                should_test=true
+            fi
         fi
         if [ "${should_test}" = true ]; then
             echo "running test in ${d}"