From 121e0cd35ef9629e5753fd5733d58b96c6cec373 Mon Sep 17 00:00:00 2001 From: Justin Griffin Date: Thu, 16 Sep 2021 11:34:36 -0400 Subject: [PATCH] General cleanup and improved output from check/in/out Now outputting more activity statements to the concourse web ui. For example, each active source filter is printed out from `check` as it is evaluated. Fixed a bug in the `log()` method that leaked param args from previous invocation to the global context. For example, ``` log "private message 1" log -p "public message 2" log "private message 3" ``` Would print to the concourse ui: ``` public message 2 private message 3 ``` Fixed this by using `local` declarations for those param args. Finally, did some general cleanup and better organization of the `common` script, and in the tests. --- assets/check | 14 +++++++------- assets/common | 28 ++++++++++++++++++++++------ assets/in | 6 +++--- assets/out | 4 ++-- test/check.bats | 30 ++++++++---------------------- 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/assets/check b/assets/check index 0da986e..28d3e00 100755 --- a/assets/check +++ b/assets/check @@ -29,15 +29,15 @@ queryForVersions() { log "${red}Warning:${reset} No namespace configured! Defaulting to ${blue}--all-namespaces${reset}." namespace_arg="--all-namespaces" fi - log "\n--> querying k8s cluster ${blue}'${source_url}'${reset} in namespace ${blue}'$(namespace)'${reset} for ${yellow}'${source_resource_types}'${reset} resources..." - new_versions=$(kubectl --server=$source_url --token=$source_token ${certificate_arg} get $source_resource_types $namespace_arg --sort-by='{.metadata.resourceVersion}' -o json | jq '[.items[]]' ) || exit $1 + log -p "\n--> querying k8s cluster ${blue}${source_url}${reset} in namespace ${blue}$(namespace)${reset} for ${yellow}${source_resource_types}${reset} resources..." + new_versions=$(kubectl --server=${source_url} --token=${source_token} ${certificate_arg} get ${source_resource_types} ${namespace_arg} --sort-by='{.metadata.resourceVersion}' -o json | jq '[.items[]]' ) || exit $1 log "$new_versions" } filterByName() { FILTER_NAME=$(jq -r '.source.filter.name // ""' < $payload) if [ ! -z "$FILTER_NAME" ]; then - log "\n--> filtering by name '$FILTER_NAME'..." + log -p "\n--> filtering by name: ${cyan}$FILTER_NAME${reset}" new_versions=$(echo "$new_versions" | tr '\r\n' ' ' | jq --arg FILTER_NAME $FILTER_NAME -r '[.[] | select(.metadata.name | test($FILTER_NAME))]') log "$new_versions" else @@ -48,7 +48,7 @@ filterByName() { filterByCreationOlderThan() { FILTER_OLDER_THAN=$(jq -r '.source.filter.olderThan // ""' < $payload) if [ ! -z "$FILTER_OLDER_THAN" ]; then - log "\n--> filtering by creation timestamp older than '$FILTER_OLDER_THAN'..." + log -p "\n--> filtering by creation timestamp older than: ${cyan}$FILTER_OLDER_THAN${reset}" new_versions=$(echo $new_versions | tr '\r\n' ' ' | jq --argjson MAX_AGE $(( $(date +%s) - $FILTER_OLDER_THAN)) -r '[.[] | select(.metadata.creationTimestamp | fromdate | tonumber | select(. < $MAX_AGE)) ]') log "$new_versions" else @@ -62,7 +62,7 @@ filterByPhases() { # remove any duplicates FILTER_PHASES=$(echo "$FILTER_PHASES" | jq 'unique') - log "\n--> filtering by phases '$FILTER_PHASES'..." + log -p "\n--> filtering by phases: ${cyan}$FILTER_PHASES${reset}" new_versions=$(echo "$new_versions" | tr '\r\n' ' ' | jq --argjson FILTER_PHASES "$FILTER_PHASES" -r '[.[] | select(.status.phase as $candidate | $FILTER_PHASES[] as $required | $candidate == $required)]') log "$new_versions" else @@ -75,10 +75,10 @@ filterByJQExpressions() { FILTER_TRANSFORM=$(jq -r '.source.filter.jq_transform // ""' < $payload) FILTER_QUERY=$(jq -r ".source.filter.jq // [] | map(\"(\" + . + \")\") | join(\" ${FILTER_OPERATOR} \") // \"\"" < $payload) if [ ! -z "$FILTER_QUERY" ]; then - log "\n--> filtering by JQ query: '$FILTER_QUERY'" + log -p "\n--> filtering by JQ query: ${cyan}$FILTER_QUERY${reset}" if [ -n "$FILTER_TRANSFORM" ]; then FILTER_TRANSFORM="| $FILTER_TRANSFORM" - log "\n with transformation: '$FILTER_TRANSFORM'" + log -p " with transformation: ${cyan}$FILTER_TRANSFORM${reset}" fi new_versions=$(echo "$new_versions" | tr '\r\n' ' ') new_versions=$(echo "$new_versions" | jq -r "[.[] | select( $FILTER_QUERY ) ] | unique $FILTER_TRANSFORM") diff --git a/assets/common b/assets/common index 93aa907..76c0799 100755 --- a/assets/common +++ b/assets/common @@ -9,17 +9,20 @@ export cyan=${esc}$(printf '[36m') export white=${esc}$(printf '[37m') export reset=${esc}$(printf '[0m') +# ------------------------------------------------------------------------------------- +# helper functions + log() { # accept args while test $# -gt 0; do case "$1" in -p|--public) shift - PUBLIC=true + local PUBLIC=true ;; -j|--json) shift - JSON=true + local JSON=true ;; *) break @@ -87,9 +90,15 @@ namespace() { echo ${namespace_arg} | sed 's/-n //g' } +# ------------------------------------------------------------------------------------- +# write the request payload to a tmp file payload=$(mktemp /tmp/resource-in.XXXXXX) cat > $payload <&0 + +# ------------------------------------------------------------------------------------- +# slurp some of the config into variables to make them easily accessible by our scripts + # source config source_url=$(jq -r '.source.url // ""' < $payload) source_token=$(jq -r '.source.token // ""' < $payload) @@ -99,15 +108,21 @@ source_namespace=$(jq -r '.source.namespace | select(.!=null)' < $payload) source_sensitive=$(jq -r '.source.sensitive | select(.!=null)' < $payload) source_insecure_skip_tls_verify=$(jq -r '.source.insecure_skip_tls_verify // false' < $payload) -# write the ca file out (have to pass a file ref to kubectl) -source_ca_file=$(mktemp /tmp/resource-ca_file.XXXXXX) -echo "$source_ca" > "$source_ca_file" - # params config params_kubectl=$(jq -r '.params.kubectl | select(.!=null)' < $payload) params_namespace=$(jq -r '.params.namespace | select(.!=null)' < $payload) params_sensitive=$(jq -r '.params.sensitive | select(.!=null)' < $payload) + +# ------------------------------------------------------------------------------------- +# write the ca file out (have to pass a file ref to kubectl) +source_ca_file=$(mktemp /tmp/resource-ca_file.XXXXXX) +echo "$source_ca" > "$source_ca_file" + + +# ------------------------------------------------------------------------------------- +# "special" crafting of some args + # configure the 'namespace_arg', with the params having highest precedence if isSet params_namespace; then namespace_arg="-n $params_namespace" @@ -115,6 +130,7 @@ elif isSet source_namespace; then namespace_arg="-n $source_namespace" fi +# if insecure ca is configured, we don't use a certificate if isTrue source_insecure_skip_tls_verify; then certificate_arg="--insecure-skip-tls-verify" else diff --git a/assets/in b/assets/in index 86fc8e4..2cfa68c 100755 --- a/assets/in +++ b/assets/in @@ -39,9 +39,9 @@ fetchResource() { fi # fetch the requested resource - log -p "\nRetrieving ${cyan}${source_resource_types}${reset} resource ${yellow}'$uid'${reset} at version ${yellow}'$resourceVersion'${reset} from cluster at ${blue}'${source_url}'${reset} in namespace ${blue}'$(namespace)'${reset}..." - kubectl --server=$source_url --token=$source_token ${certificate_arg} get $source_resource_types $namespace_arg --sort-by={.metadata.resourceVersion} -o json \ - | jq --arg uid $uid --arg resourceVersion $resourceVersion '.items[] | select((.metadata.uid == $uid and .metadata.resourceVersion == $resourceVersion))' \ + log -p "\n--> Retrieving ${yellow}${source_resource_types}${reset} resource ${cyan}$uid${reset} at version ${cyan}$resourceVersion${reset} from cluster at ${blue}${source_url}${reset} in namespace ${blue}$(namespace)${reset}..." + kubectl --server=${source_url} --token=${source_token} ${certificate_arg} get ${source_resource_types} ${namespace_arg} --sort-by={.metadata.resourceVersion} -o json \ + | jq --arg uid ${uid} --arg resourceVersion ${resourceVersion} '.items[] | select((.metadata.uid == $uid and .metadata.resourceVersion == $resourceVersion))' \ > $target_dir/resource.json } diff --git a/assets/out b/assets/out index 749b586..5801748 100755 --- a/assets/out +++ b/assets/out @@ -23,8 +23,8 @@ invokeKubectl() { log -p "${red}Warning:${reset} No namespace configured!" fi - log -p "\nInvoking ${cyan}kubectl${reset} with args ${yellow}'${params_kubectl}'${reset} targeting cluster at ${blue}'${source_url}'${reset} in namespace ${blue}'$(namespace)'${reset}..." - kubectl --server=$source_url --token=$source_token ${certificate_arg} ${namespace_arg} ${params_kubectl} + log -p "\n--> Invoking ${cyan}kubectl${reset} with args ${yellow}${params_kubectl}${reset} targeting cluster at ${blue}${source_url}${reset} in namespace ${blue}$(namespace)${reset}..." + kubectl --server=${source_url} --token=${source_token} ${certificate_arg} ${namespace_arg} ${params_kubectl} } emitResult() { diff --git a/test/check.bats b/test/check.bats index 86cb6ff..81609be 100644 --- a/test/check.bats +++ b/test/check.bats @@ -4,10 +4,9 @@ load '/opt/bats/addons/bats-support/load.bash' load '/opt/bats/addons/bats-assert/load.bash' load '/opt/bats/addons/bats-mock/stub.bash' -setup() { - # this is the default when source config doesn't specify - expected_namespace_arg="--all-namespaces" -} +#setup() { + # do any general setup +#} source_check() { stdin_payload=${1:-"stdin-source"} @@ -22,8 +21,8 @@ source_check() { export -f log # mock kubectl to return our expected response - local expected_kubectl_args="--server=$source_url --token=$source_token --certificate-authority=$source_ca_file \ - get $source_resource_types $expected_namespace_arg --sort-by={.metadata.resourceVersion} -o json" + local expected_kubectl_args="--server=$source_url --token=$source_token ${expected_ca_arg:---certificate-authority=$source_ca_file} \ + get $source_resource_types ${expected_namespace_arg:---all-namespaces} --sort-by={.metadata.resourceVersion} -o json" if [ $kubectl_response == "FAIL" ]; then stub kubectl "$expected_kubectl_args : exit 1" @@ -50,7 +49,7 @@ teardown() { } @test "[check] previous version is empty if not provided" { - source_check "stdin-source" + source_check extractPreviousVersion @@ -699,22 +698,9 @@ teardown() { } @test "[check] GH-20 uses '--insecure-skip-tls-verify' when 'source.insecure_skip_tls_verify' is 'true'" { - # source the common script - source "$SUT_ASSETS_DIR/common" <<< "$(<$BATS_TEST_DIRNAME/fixtures/stdin-source-insecure-skip-tls-verify-true.json)" - - # stub the log function - log() { :; } - export -f log - - # mock kubectl to expect to be called with '--insecure-skip-tls-verify' - local expected_kubectl_args="--server=$source_url --token=$source_token --insecure-skip-tls-verify \ - get $source_resource_types $expected_namespace_arg --sort-by={.metadata.resourceVersion} -o json" - stub kubectl "$expected_kubectl_args : cat $BATS_TEST_DIRNAME/fixtures/kubectl-response.json" - - # source the sut - source "$SUT_ASSETS_DIR/check" + expected_ca_arg="--insecure-skip-tls-verify" + source_check "stdin-source-insecure-skip-tls-verify-true" - # run the test queryForVersions assert_equal $(jq length <<< "$new_versions") 3