From b500045c29c3b6b35a08e44482d76ed562b73d4e Mon Sep 17 00:00:00 2001 From: Marc Boudreau Date: Thu, 30 Mar 2023 11:00:02 -0400 Subject: [PATCH] Backport all GHA Migration Changes to release/1.13.x Branch (#19767) * backport all gha migration changes to release/1.13.x * remove the .circleci directory * remove references to circleci configuration from pre-commit hook * remove reference to .circleci in Makefile * port change to how gofumpt is executed in Makefile * add gotestsum to tools/tools.go * remove postgresql/scram package from generate-test-package-lists.sh since it didn't exist in release 1.13 or earlier * blank out environment variables to allow test to properly function * use go:embed to load files into test --------- Co-authored-by: Kuba Wieczorek --- .circleci/.gitattributes | 1 - .circleci/.gitignore | 1 - .circleci/Makefile | 100 -- .circleci/README.md | 130 -- .circleci/config.yml | 1216 ----------------- .circleci/config/@config.yml | 5 - .circleci/config/commands/@caches.yml | 59 - .circleci/config/commands/configure-git.yml | 7 - .../exit-if-branch-does-not-need-test-ui.yml | 17 - .../commands/exit-if-ui-or-docs-branch.yml | 14 - .circleci/config/commands/go_test.yml | 226 --- .circleci/config/commands/setup-go.yml | 35 - .circleci/config/executors/@executors.yml | 49 - .circleci/config/jobs/build-go-dev.yml | 20 - .circleci/config/jobs/fmt.yml | 17 - .../config/jobs/install-ui-dependencies.yml | 11 - .circleci/config/jobs/pre-flight-checks.yml | 34 - .circleci/config/jobs/semgrep.yml | 15 - .circleci/config/jobs/test-go-nightly.yml | 14 - .../jobs/test-go-race-remote-docker.yml | 18 - .circleci/config/jobs/test-go-race.yml | 14 - .../config/jobs/test-go-remote-docker.yml | 17 - .circleci/config/jobs/test-go.yml | 13 - .circleci/config/jobs/test-ui.yml | 22 - .circleci/config/workflows/ci.yml | 35 - .github/actionlint.yaml | 10 + .../scripts/generate-test-package-lists.sh | 283 ++++ .../test-generate-test-package-lists.sh | 75 + .github/workflows/actionlint.yml | 15 + .github/workflows/build-vault-oss.yml | 6 +- .github/workflows/build.yml | 15 +- .github/workflows/changelog-checker.yml | 4 +- .github/workflows/ci.yml | 266 ++++ .../workflows/enos-release-testing-oss.yml | 7 +- .github/workflows/enos-run-k8s.yml | 12 +- .github/workflows/godoc-test-checker.yml | 12 +- .github/workflows/goversion-checker.yml | 23 - .github/workflows/oss.yml | 12 +- .github/workflows/security-scan.yml | 19 +- .github/workflows/setup-go-cache.yml | 33 + .github/workflows/test-ci-cleanup.yml | 2 +- .github/workflows/test-enos-scenario-ui.yml | 16 +- .github/workflows/test-go.yml | 180 +++ .../workflows/test-run-acc-tests-for-path.yml | 10 +- .../test-run-enos-scenario-matrix.yml | 8 +- .hooks/pre-commit | 80 +- Makefile | 9 +- command/server/hcp_link_config_test.go | 4 + go.mod | 3 + go.sum | 17 +- .../kubernetes/testing/testserver.go | 49 +- tools/tools.go | 3 + 52 files changed, 968 insertions(+), 2295 deletions(-) delete mode 100644 .circleci/.gitattributes delete mode 100644 .circleci/.gitignore delete mode 100644 .circleci/Makefile delete mode 100644 .circleci/README.md delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/config/@config.yml delete mode 100644 .circleci/config/commands/@caches.yml delete mode 100644 .circleci/config/commands/configure-git.yml delete mode 100644 .circleci/config/commands/exit-if-branch-does-not-need-test-ui.yml delete mode 100644 .circleci/config/commands/exit-if-ui-or-docs-branch.yml delete mode 100644 .circleci/config/commands/go_test.yml delete mode 100644 .circleci/config/commands/setup-go.yml delete mode 100644 .circleci/config/executors/@executors.yml delete mode 100644 .circleci/config/jobs/build-go-dev.yml delete mode 100644 .circleci/config/jobs/fmt.yml delete mode 100644 .circleci/config/jobs/install-ui-dependencies.yml delete mode 100644 .circleci/config/jobs/pre-flight-checks.yml delete mode 100644 .circleci/config/jobs/semgrep.yml delete mode 100644 .circleci/config/jobs/test-go-nightly.yml delete mode 100644 .circleci/config/jobs/test-go-race-remote-docker.yml delete mode 100644 .circleci/config/jobs/test-go-race.yml delete mode 100644 .circleci/config/jobs/test-go-remote-docker.yml delete mode 100644 .circleci/config/jobs/test-go.yml delete mode 100644 .circleci/config/jobs/test-ui.yml delete mode 100644 .circleci/config/workflows/ci.yml create mode 100644 .github/actionlint.yaml create mode 100755 .github/scripts/generate-test-package-lists.sh create mode 100755 .github/scripts/test-generate-test-package-lists.sh create mode 100644 .github/workflows/actionlint.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/goversion-checker.yml create mode 100644 .github/workflows/setup-go-cache.yml create mode 100644 .github/workflows/test-go.yml diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes deleted file mode 100644 index 2dd06ee5f7cd..000000000000 --- a/.circleci/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -config.yml linguist-generated diff --git a/.circleci/.gitignore b/.circleci/.gitignore deleted file mode 100644 index 3018b3a68132..000000000000 --- a/.circleci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.tmp/ diff --git a/.circleci/Makefile b/.circleci/Makefile deleted file mode 100644 index dc75ea5f1f19..000000000000 --- a/.circleci/Makefile +++ /dev/null @@ -1,100 +0,0 @@ -# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility. -# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/ -SHELL := /usr/bin/env bash -euo pipefail -c - -# CONFIG is the name of the make target someone -# would invoke to update the main config file (config.yml). -CONFIG ?= ci-config -# VERIFY is the name of the make target someone -# would invoke to verify the config file. -VERIFY ?= ci-verify - -CIRCLECI := circleci --skip-update-check -ifeq ($(DEBUG_CIRCLECI_CLI),YES) -CIRCLECI += --debug -endif - -# For config processing, always refer to circleci.com not self-hosted circleci, -# because self-hosted does not currently support the necessary API. -CIRCLECI_CLI_HOST := https://circleci.com -export CIRCLECI_CLI_HOST - -# Set up some documentation/help message variables. -# We do not attempt to install the CircleCI CLI from this Makefile. -CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation -CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK) -CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null) -ifeq ($(CCI_VERSION),) -# Attempting to use the CLI fails with installation instructions. -CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \# -endif - -SOURCE_DIR := config -SOURCE_YML := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml') -CONFIG_SOURCE := Makefile $(SOURCE_YML) | $(SOURCE_DIR) -OUT := config.yml -TMP := .tmp/config-processed -CONFIG_PACKED := .tmp/config-packed -GO_VERSION_FILE := ../.go-version -GO_VERSION := $(shell cat $(GO_VERSION_FILE)) - -default: help - -help: - @echo "Usage:" - @echo " make $(CONFIG): recompile config.yml from $(SOURCE_DIR)/" - @echo " make $(VERIFY): verify that config.yml is a true mapping from $(SOURCE_DIR)/" - @echo - @echo "Diagnostics:" - @[ -z "$(CCI_VERSION)" ] || echo " circleci-cli version $(CCI_VERSION)" - @[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)" - -$(SOURCE_DIR): - @echo No source directory $(SOURCE_DIR) found.; exit 1 - -# Make sure our .tmp dir exists. -$(shell [ -d .tmp ] || mkdir .tmp) - -.PHONY: $(CONFIG) -$(CONFIG): $(OUT) $(GO_VERSION_FILE) - -.PHONY: $(VERIFY) -$(VERIFY): config-up-to-date - @$(CIRCLECI) config validate $(OUT) - -define GENERATED_FILE_HEADER -### *** -### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make $(CONFIG)'. -### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make $(CONFIG)'. -### *** -endef -export GENERATED_FILE_HEADER - -# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds, -# it them moves that file to $@. This makes is an atomic operation, so if it fails -# make doesn't consider a half-baked file up to date. -define GEN_CONFIG - @yq -i ".references.environment.GO_IMAGE = \"docker.mirror.hashicorp.services/cimg/go:$(GO_VERSION)\"" $(SOURCE_DIR)/executors/\@executors.yml - - @$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED) - @echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; } - @$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; } - @mv -f $@.tmp $@ -endef - -.PHONY: $(OUT) -$(OUT): $(CONFIG_SOURCE) - $(GEN_CONFIG) - @echo "$@ updated" - -$(TMP): $(CONFIG_SOURCE) - $(GEN_CONFIG) - -.PHONY: config-up-to-date -config-up-to-date: $(TMP) # Note this must not depend on $(OUT)! - @if diff -w $(OUT) $<; then \ - echo "Generated $(OUT) is up to date!"; \ - else \ - echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \ - exit 1; \ - fi diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index 1ec75cafade9..000000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# How to use CircleCI multi-file config - -This README and the Makefile should be in your `.circleci` directory, -in the root of your repository. -All path references in this README assume we are in this `.circleci` directory. - -The `Makefile` in this directory generates `./config.yml` in CircleCI 2.0 syntax, -from the tree rooted at `./config/`, which contains files in CircleCI 2.0 or 2.1 syntax. - - -## Quickstart - -The basic workflow is: - -- Edit source files in `./config/` -- When you are done, run `make ci-config` to update `./config.yml` -- Commit this entire `.circleci` directory, including that generated file together. -- Run `make ci-verify` to ensure the current `./config.yml` is up to date with the source. - -When merging this `.circleci` directory: - -- Do not merge the generated `./config.yml` file, instead: -- Merge the source files under `./config/`, and then -- Run `make ci-config` to re-generate the merged `./config.yml` - -And that's it, for more detail, read on! - - -## How does it work, roughly? - -CircleCI supports [generating a single config file from many], -using the `$ circleci config pack` command. -It also supports [expanding 2.1 syntax to 2.0 syntax] -using the `$ circleci config process` command. -We use these two commands, stitched together using the `Makefile` -to implement the workflow. - -[generating a single config file from many]: https://circleci.com/docs/2.0/local-cli/#packing-a-config -[expanding 2.1 syntax to 2.0 syntax]: https://circleci.com/docs/2.0/local-cli/#processing-a-config - - -## Prerequisites - -You will need the [CircleCI CLI tool] installed and working, -at least version `0.1.5607`. -You can [download this tool directly from GitHub Releases]. - -``` -$ circleci version -0.1.5607+f705856 -``` - -[CircleCI CLI tool]: https://circleci.com/docs/2.0/local-cli/ -[download this tool directly from GitHub Releases]: https://github.com/CircleCI-Public/circleci-cli/releases - - -## Updating the config source - -Before making changes, be sure to understand the layout -of the `./config/` file tree, as well as circleci 2.1 syntax. -See the [Syntax and layout] section below. - -To update the config, you should edit, add or remove files -in the `./config/` directory, -and then run `make ci-config`. -If that's successful, -you should then commit every `*.yml` file in the tree rooted in this directory. -That is: you should commit both the source under `./config/` -and the generated file `./config.yml` at the same time, in the same commit. -The included git pre-commit hook will help with this. -Do not edit the `./config.yml` file directly, as you will lose your changes -next time `make ci-config` is run. - -[Syntax and layout]: #syntax-and-layout - - -### Verifying `./config.yml` - -To check whether or not the current `./config.yml` is up to date with the source -and valid, run `$ make ci-verify`. -Note that `$ make ci-verify` should be run in CI, -in case not everyone has the git pre-commit hook set up correctly. - - -#### Example shell session - -```sh -$ make ci-config -config.yml updated -$ git add -A . # The -A makes sure to include deletions/renames etc. -$ git commit -m "ci: blah blah blah" -Changes detected in .circleci/, running 'make -C .circleci ci-verify' ---> Generated config.yml is up to date! ---> Config file at config.yml is valid. -``` - - -### Syntax and layout - -It is important to understand the layout of the config directory. -Read the documentation on [packing a config] for a full understanding -of how multiple YAML files are merged by the circleci CLI tool. - -[packing a config]: https://circleci.com/docs/2.0/local-cli/#packing-a-config - -Here is an example file tree (with comments added afterwards): - -```sh -$ tree . -. -├── Makefile -├── README.md # This file. -├── config # The source code for config.yml is rooted here. -│   ├── @config.yml # Files beginning with @ are treated specially by `circleci config pack` -│   ├── commands # Subdirectories of config become top-level keys. -│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under -│   │   └── go_build.yml # their parent (in this case "commands"). -│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: -│   └── jobs # jobs also becomes a top-level key under config... -│   ├── build.yml # ...and likewise filenames become keys under their parent. -│   └── test.yml -└── config.yml # The generated file in 2.0 syntax. -``` - -About those `@` files... Preceding a filename with `@` -indicates to `$ circleci config pack` that the contents of this YAML file -should be at the top-level, rather than underneath a key named after their filename. -This naming convention is unfortunate as it breaks autocompletion in bash, -but there we go. - diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3ae52c52e856..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1216 +0,0 @@ -### *** -### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. -### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'. -### *** -# Orb 'circleci/slack@3.2.0' resolved to 'circleci/slack@3.2.0' -version: 2 -jobs: - install-ui-dependencies: - docker: - - environment: - JOBS: 2 - image: docker.mirror.hashicorp.services/circleci/node:14-browsers - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - steps: - - checkout - - restore_cache: - key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }} - name: Restore yarn cache - - run: - command: | - cd ui - yarn install - npm rebuild node-sass - name: Install UI dependencies - - save_cache: - key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }} - name: Save yarn cache - paths: - - ui/node_modules - test-ui: - docker: - - environment: - JOBS: 2 - image: docker.mirror.hashicorp.services/circleci/node:14-browsers - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - resource_class: xlarge - steps: - - run: - command: | - case "$CIRCLE_BRANCH" in - main|ui/*|backport/ui/*|release/*|merge*) ;; - *) # If the branch being tested doesn't match one of the above patterns, - # we don't need to run test-ui and can abort the job. - circleci-agent step halt - ;; - esac - - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - - checkout - - restore_cache: - key: yarn-lock-v7-{{ checksum "ui/yarn.lock" }} - name: Restore yarn cache - - attach_workspace: - at: . - - run: - command: | - # Add ./bin to the PATH so vault binary can be run by Ember tests - export PATH="${PWD}/bin:${PATH}" - - # Run Ember tests - cd ui - mkdir -p test-results/qunit - yarn test:oss - name: Test UI - - store_artifacts: - path: ui/test-results - - store_test_results: - path: ui/test-results - build-go-dev: - machine: - image: ubuntu-2004:2022.10.1 - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - steps: - - checkout - - run: - command: | - GO_VERSION=$(cat .go-version) - [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } - # Install Go - cd ~ - curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" - sudo rm -rf /usr/local/go - sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - rm -f "go${GO_VERSION}.linux-amd64.tar.gz" - GOPATH="/home/circleci/go" - mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } - mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; } - echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" - echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" - echo "export GOPROXY=off" >> "$BASH_ENV" - echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV" - - echo "$ go version" - go version - name: Setup Go - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Restore exact go modules cache - - attach_workspace: - at: . - - run: - command: | - # Move dev UI assets to expected location - rm -rf ./pkg - mkdir ./pkg - - # Build dev binary - make ci-bootstrap dev - name: Build dev binary - - persist_to_workspace: - paths: - - bin - root: . - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - test-go-remote-docker: - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.20.1 - resource_class: medium - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - parallelism: 8 - steps: - - run: - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 20.10.17 - - add_ssh_keys: - fingerprints: - - b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9 - - run: - command: | - git config --global url."git@github.com:".insteadOf https://github.com/ - - run: - command: | - TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key - name: Compute test cache key - - restore_cache: - keys: - - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Restore exact go modules cache - - run: - command: | - set -exo pipefail - - EXTRA_TAGS= - case "" in - *-race*) export VAULT_CI_GO_TEST_RACE=1;; - *) EXTRA_TAGS=deadlock;; - esac - - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - USE_DOCKER=0 - USE_DOCKER=1 - - # Check all directories with a go.mod file - modules=("." "api" "sdk") - all_package_names="" - - for dir in "${modules[@]}" - do - pushd "$dir" - # On its own line so that -e will fail the tests if we detect errors here. - go list -test -json ./... > test-list.json - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - # The appended true condition ensures the command will succeed if no packages are found - if [ $USE_DOCKER == 1 ]; then - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - else - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - fi - # Move back into root directory - popd - # Append the test packages into the global list, if any are found - if [ -n "$package_names" ]; then - all_package_names+=" ${package_names}" - fi - done - - # After running tests split step, we are now running the following steps - # in multiple different containers, each getting a different subset of - # the test packages in their package_names variable. Each container - # has its own remote docker VM. - - make prep - - mkdir -p test-results/go-test - - # We don't want VAULT_LICENSE set when running Go tests, because that's - # not what developers have in their environments and it could break some - # tests; it would be like setting VAULT_TOKEN. However some non-Go - # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we set VAULT_LICENSE in CI, and here we unset it. Instead of - # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want - # an externally supplied license can opt-in to using it. - export VAULT_LICENSE_CI="$VAULT_LICENSE" - VAULT_LICENSE= - - # Create a docker network for our test container - if [ $USE_DOCKER == 1 ]; then - # Despite the fact that we're using a circleci image (thus getting the - # version they chose for the docker cli) and that we're specifying a - # docker version to use for the remote docker instances, we occasionally - # see "client version too new, max supported version 1.39" errors for - # reasons unclear. - export DOCKER_API_VERSION=1.39 - - TEST_DOCKER_NETWORK_NAME="${CIRCLE_WORKFLOW_JOB_ID}-${CIRCLE_NODE_INDEX}" - export TEST_DOCKER_NETWORK_ID=$(docker network list --quiet --no-trunc --filter="name=${TEST_DOCKER_NETWORK_NAME}") - if [ -z $TEST_DOCKER_NETWORK_ID ]; then - docker network prune -f - TEST_DOCKER_NETWORK_ID=$(docker network create "${TEST_DOCKER_NETWORK_NAME}") - fi - - - - # Start a docker test container to run the tests in - CONTAINER_ID="$(docker run -d \ - -e TEST_DOCKER_NETWORK_ID \ - -e GOPRIVATE \ - -e DOCKER_CERT_PATH \ - -e DOCKER_HOST \ - -e DOCKER_MACHINE_NAME \ - -e DOCKER_TLS_VERIFY \ - -e NO_PROXY \ - -e VAULT_TEST_LOG_DIR=/tmp/testlogs \ - --network ${TEST_DOCKER_NETWORK_NAME} \ - $GO_IMAGE \ - tail -f /dev/null)" - mkdir workspace - echo ${CONTAINER_ID} > workspace/container_id - - # Hack: Docker permissions appear to have changed; let's explicitly - # add a new user/group with the correct host uid to the docker - # container, fixing all of these permissions issues correctly. We - # then have to run with this user consistently in the future. - # - # Notably, in this shell pipeline we see: - # uid=1001(circleci) gid=1002(circleci) groups=1002(circleci) - # - # but inside the docker image below, we see: - # uid=3434(circleci) gid=3434(circleci) groups=3434(circleci) - # - # See also: https://github.com/CircleCI-Public/cimg-base/issues/122 - export HOST_GID="$(id -g)" - export HOST_UID="$(id -u)" - export CONT_GID="$(docker exec ${CONTAINER_ID} sh -c 'id -g')" - export CONT_GNAME="$(docker exec ${CONTAINER_ID} sh -c 'id -g -n')" - export CONT_UID="$(docker exec ${CONTAINER_ID} sh -c 'id -u')" - if (( HOST_UID != CONT_UID )); then - # Only provision a group if necessary; otherwise reuse the - # existing one. - if (( HOST_GID != CONT_GID )); then - docker exec -e HOST_GID -e CONT_GNAME ${CONTAINER_ID} sh -c 'sudo groupmod -g $HOST_GID $CONT_GNAME' - fi - - docker exec -e CONT_GNAME -e HOST_UID ${CONTAINER_ID} sh -c 'sudo usermod -a -G $CONT_GNAME -u $HOST_UID circleci' - fi - - # Run tests - test -d /tmp/go-cache && docker cp /tmp/go-cache ${CONTAINER_ID}:/tmp/gocache - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault' - docker cp . ${CONTAINER_ID}:/home/circleci/go/src/github.com/hashicorp/vault/ - docker cp $DOCKER_CERT_PATH/ ${CONTAINER_ID}:$DOCKER_CERT_PATH - - # Copy the downloaded modules inside the container. - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/pkg' - docker cp "$(go env GOPATH)/pkg/mod" ${CONTAINER_ID}:/home/circleci/go/pkg/mod - - docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \ - -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \ - -e GOCACHE=/tmp/gocache \ - -e GO_TAGS \ - -e GOPROXY="off" \ - -e VAULT_LICENSE_CI \ - -e GOARCH=amd64 \ - ${CONTAINER_ID} \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - \ - ${all_package_names} - else - GOARCH=amd64 \ - GOCACHE=/tmp/go-cache \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - \ - ${all_package_names} - fi - environment: - GOPRIVATE: github.com/hashicorp/* - name: Run Go tests - no_output_timeout: 60m - - run: - command: | - docker cp $(cat workspace/container_id):/home/circleci/go/src/github.com/hashicorp/vault/test-results . - docker cp $(cat workspace/container_id):/tmp/gocache /tmp/go-cache - name: Copy test results - when: always - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: /tmp/testlogs - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - fmt: - machine: - image: ubuntu-2004:2022.10.1 - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - steps: - - checkout - - run: - command: | - GO_VERSION=$(cat .go-version) - [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } - # Install Go - cd ~ - curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" - sudo rm -rf /usr/local/go - sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - rm -f "go${GO_VERSION}.linux-amd64.tar.gz" - GOPATH="/home/circleci/go" - mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } - mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; } - echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" - echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" - echo "export GOPROXY=https://proxy.golang.org,direct" >> "$BASH_ENV" - echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV" - - echo "$ go version" - go version - name: Setup Go - - run: - command: | - echo "Using gofumpt version ${GOFUMPT_VERSION}" - go install "mvdan.cc/gofumpt@v${GOFUMPT_VERSION}" - make fmt - if ! git diff --exit-code; then - echo "Code has formatting errors. Run 'make fmt' to fix" - exit 1 - fi - name: make fmt - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - test-go-race: - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.20.1 - resource_class: xlarge - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - parallelism: 8 - steps: - - run: - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - - checkout - - add_ssh_keys: - fingerprints: - - b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9 - - run: - command: | - git config --global url."git@github.com:".insteadOf https://github.com/ - - run: - command: | - TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key - name: Compute test cache key - - restore_cache: - keys: - - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Restore exact go modules cache - - run: - command: | - set -exo pipefail - - EXTRA_TAGS= - case "-race" in - *-race*) export VAULT_CI_GO_TEST_RACE=1;; - *) EXTRA_TAGS=deadlock;; - esac - - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - USE_DOCKER=0 - - # Check all directories with a go.mod file - modules=("." "api" "sdk") - all_package_names="" - - for dir in "${modules[@]}" - do - pushd "$dir" - # On its own line so that -e will fail the tests if we detect errors here. - go list -test -json ./... > test-list.json - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - # The appended true condition ensures the command will succeed if no packages are found - if [ $USE_DOCKER == 1 ]; then - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - else - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - fi - # Move back into root directory - popd - # Append the test packages into the global list, if any are found - if [ -n "$package_names" ]; then - all_package_names+=" ${package_names}" - fi - done - - # After running tests split step, we are now running the following steps - # in multiple different containers, each getting a different subset of - # the test packages in their package_names variable. Each container - # has its own remote docker VM. - - make prep - - mkdir -p test-results/go-test - - # We don't want VAULT_LICENSE set when running Go tests, because that's - # not what developers have in their environments and it could break some - # tests; it would be like setting VAULT_TOKEN. However some non-Go - # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we set VAULT_LICENSE in CI, and here we unset it. Instead of - # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want - # an externally supplied license can opt-in to using it. - export VAULT_LICENSE_CI="$VAULT_LICENSE" - VAULT_LICENSE= - - # Create a docker network for our test container - if [ $USE_DOCKER == 1 ]; then - # Despite the fact that we're using a circleci image (thus getting the - # version they chose for the docker cli) and that we're specifying a - # docker version to use for the remote docker instances, we occasionally - # see "client version too new, max supported version 1.39" errors for - # reasons unclear. - export DOCKER_API_VERSION=1.39 - - TEST_DOCKER_NETWORK_NAME="${CIRCLE_WORKFLOW_JOB_ID}-${CIRCLE_NODE_INDEX}" - export TEST_DOCKER_NETWORK_ID=$(docker network list --quiet --no-trunc --filter="name=${TEST_DOCKER_NETWORK_NAME}") - if [ -z $TEST_DOCKER_NETWORK_ID ]; then - docker network prune -f - TEST_DOCKER_NETWORK_ID=$(docker network create "${TEST_DOCKER_NETWORK_NAME}") - fi - - - - # Start a docker test container to run the tests in - CONTAINER_ID="$(docker run -d \ - -e TEST_DOCKER_NETWORK_ID \ - -e GOPRIVATE \ - -e DOCKER_CERT_PATH \ - -e DOCKER_HOST \ - -e DOCKER_MACHINE_NAME \ - -e DOCKER_TLS_VERIFY \ - -e NO_PROXY \ - -e VAULT_TEST_LOG_DIR=/tmp/testlogs \ - --network ${TEST_DOCKER_NETWORK_NAME} \ - $GO_IMAGE \ - tail -f /dev/null)" - mkdir workspace - echo ${CONTAINER_ID} > workspace/container_id - - # Hack: Docker permissions appear to have changed; let's explicitly - # add a new user/group with the correct host uid to the docker - # container, fixing all of these permissions issues correctly. We - # then have to run with this user consistently in the future. - # - # Notably, in this shell pipeline we see: - # uid=1001(circleci) gid=1002(circleci) groups=1002(circleci) - # - # but inside the docker image below, we see: - # uid=3434(circleci) gid=3434(circleci) groups=3434(circleci) - # - # See also: https://github.com/CircleCI-Public/cimg-base/issues/122 - export HOST_GID="$(id -g)" - export HOST_UID="$(id -u)" - export CONT_GID="$(docker exec ${CONTAINER_ID} sh -c 'id -g')" - export CONT_GNAME="$(docker exec ${CONTAINER_ID} sh -c 'id -g -n')" - export CONT_UID="$(docker exec ${CONTAINER_ID} sh -c 'id -u')" - if (( HOST_UID != CONT_UID )); then - # Only provision a group if necessary; otherwise reuse the - # existing one. - if (( HOST_GID != CONT_GID )); then - docker exec -e HOST_GID -e CONT_GNAME ${CONTAINER_ID} sh -c 'sudo groupmod -g $HOST_GID $CONT_GNAME' - fi - - docker exec -e CONT_GNAME -e HOST_UID ${CONTAINER_ID} sh -c 'sudo usermod -a -G $CONT_GNAME -u $HOST_UID circleci' - fi - - # Run tests - test -d /tmp/go-cache && docker cp /tmp/go-cache ${CONTAINER_ID}:/tmp/gocache - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault' - docker cp . ${CONTAINER_ID}:/home/circleci/go/src/github.com/hashicorp/vault/ - docker cp $DOCKER_CERT_PATH/ ${CONTAINER_ID}:$DOCKER_CERT_PATH - - # Copy the downloaded modules inside the container. - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/pkg' - docker cp "$(go env GOPATH)/pkg/mod" ${CONTAINER_ID}:/home/circleci/go/pkg/mod - - docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \ - -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \ - -e GOCACHE=/tmp/gocache \ - -e GO_TAGS \ - -e GOPROXY="off" \ - -e VAULT_LICENSE_CI \ - -e GOARCH=amd64 \ - ${CONTAINER_ID} \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - -race \ - ${all_package_names} - else - GOARCH=amd64 \ - GOCACHE=/tmp/go-cache \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - -race \ - ${all_package_names} - fi - environment: - GOPRIVATE: github.com/hashicorp/* - name: Run Go tests - no_output_timeout: 60m - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: /tmp/testlogs - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - test-go: - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.20.1 - resource_class: large - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - parallelism: 8 - steps: - - run: - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - - checkout - - add_ssh_keys: - fingerprints: - - b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9 - - run: - command: | - git config --global url."git@github.com:".insteadOf https://github.com/ - - run: - command: | - TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key - name: Compute test cache key - - restore_cache: - keys: - - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Restore exact go modules cache - - run: - command: | - set -exo pipefail - - EXTRA_TAGS= - case "" in - *-race*) export VAULT_CI_GO_TEST_RACE=1;; - *) EXTRA_TAGS=deadlock;; - esac - - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - USE_DOCKER=0 - - # Check all directories with a go.mod file - modules=("." "api" "sdk") - all_package_names="" - - for dir in "${modules[@]}" - do - pushd "$dir" - # On its own line so that -e will fail the tests if we detect errors here. - go list -test -json ./... > test-list.json - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - # The appended true condition ensures the command will succeed if no packages are found - if [ $USE_DOCKER == 1 ]; then - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - else - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - fi - # Move back into root directory - popd - # Append the test packages into the global list, if any are found - if [ -n "$package_names" ]; then - all_package_names+=" ${package_names}" - fi - done - - # After running tests split step, we are now running the following steps - # in multiple different containers, each getting a different subset of - # the test packages in their package_names variable. Each container - # has its own remote docker VM. - - make prep - - mkdir -p test-results/go-test - - # We don't want VAULT_LICENSE set when running Go tests, because that's - # not what developers have in their environments and it could break some - # tests; it would be like setting VAULT_TOKEN. However some non-Go - # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we set VAULT_LICENSE in CI, and here we unset it. Instead of - # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want - # an externally supplied license can opt-in to using it. - export VAULT_LICENSE_CI="$VAULT_LICENSE" - VAULT_LICENSE= - - # Create a docker network for our test container - if [ $USE_DOCKER == 1 ]; then - # Despite the fact that we're using a circleci image (thus getting the - # version they chose for the docker cli) and that we're specifying a - # docker version to use for the remote docker instances, we occasionally - # see "client version too new, max supported version 1.39" errors for - # reasons unclear. - export DOCKER_API_VERSION=1.39 - - TEST_DOCKER_NETWORK_NAME="${CIRCLE_WORKFLOW_JOB_ID}-${CIRCLE_NODE_INDEX}" - export TEST_DOCKER_NETWORK_ID=$(docker network list --quiet --no-trunc --filter="name=${TEST_DOCKER_NETWORK_NAME}") - if [ -z $TEST_DOCKER_NETWORK_ID ]; then - docker network prune -f - TEST_DOCKER_NETWORK_ID=$(docker network create "${TEST_DOCKER_NETWORK_NAME}") - fi - - - - # Start a docker test container to run the tests in - CONTAINER_ID="$(docker run -d \ - -e TEST_DOCKER_NETWORK_ID \ - -e GOPRIVATE \ - -e DOCKER_CERT_PATH \ - -e DOCKER_HOST \ - -e DOCKER_MACHINE_NAME \ - -e DOCKER_TLS_VERIFY \ - -e NO_PROXY \ - -e VAULT_TEST_LOG_DIR=/tmp/testlogs \ - --network ${TEST_DOCKER_NETWORK_NAME} \ - $GO_IMAGE \ - tail -f /dev/null)" - mkdir workspace - echo ${CONTAINER_ID} > workspace/container_id - - # Hack: Docker permissions appear to have changed; let's explicitly - # add a new user/group with the correct host uid to the docker - # container, fixing all of these permissions issues correctly. We - # then have to run with this user consistently in the future. - # - # Notably, in this shell pipeline we see: - # uid=1001(circleci) gid=1002(circleci) groups=1002(circleci) - # - # but inside the docker image below, we see: - # uid=3434(circleci) gid=3434(circleci) groups=3434(circleci) - # - # See also: https://github.com/CircleCI-Public/cimg-base/issues/122 - export HOST_GID="$(id -g)" - export HOST_UID="$(id -u)" - export CONT_GID="$(docker exec ${CONTAINER_ID} sh -c 'id -g')" - export CONT_GNAME="$(docker exec ${CONTAINER_ID} sh -c 'id -g -n')" - export CONT_UID="$(docker exec ${CONTAINER_ID} sh -c 'id -u')" - if (( HOST_UID != CONT_UID )); then - # Only provision a group if necessary; otherwise reuse the - # existing one. - if (( HOST_GID != CONT_GID )); then - docker exec -e HOST_GID -e CONT_GNAME ${CONTAINER_ID} sh -c 'sudo groupmod -g $HOST_GID $CONT_GNAME' - fi - - docker exec -e CONT_GNAME -e HOST_UID ${CONTAINER_ID} sh -c 'sudo usermod -a -G $CONT_GNAME -u $HOST_UID circleci' - fi - - # Run tests - test -d /tmp/go-cache && docker cp /tmp/go-cache ${CONTAINER_ID}:/tmp/gocache - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault' - docker cp . ${CONTAINER_ID}:/home/circleci/go/src/github.com/hashicorp/vault/ - docker cp $DOCKER_CERT_PATH/ ${CONTAINER_ID}:$DOCKER_CERT_PATH - - # Copy the downloaded modules inside the container. - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/pkg' - docker cp "$(go env GOPATH)/pkg/mod" ${CONTAINER_ID}:/home/circleci/go/pkg/mod - - docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \ - -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \ - -e GOCACHE=/tmp/gocache \ - -e GO_TAGS \ - -e GOPROXY="off" \ - -e VAULT_LICENSE_CI \ - -e GOARCH=amd64 \ - ${CONTAINER_ID} \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - \ - ${all_package_names} - else - GOARCH=amd64 \ - GOCACHE=/tmp/go-cache \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - \ - ${all_package_names} - fi - environment: - GOPRIVATE: github.com/hashicorp/* - name: Run Go tests - no_output_timeout: 60m - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: /tmp/testlogs - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - semgrep: - docker: - - image: docker.mirror.hashicorp.services/returntocorp/semgrep:0.113.0 - shell: /bin/sh - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - steps: - - checkout - - attach_workspace: - at: . - - run: - command: "# Alpine images can't run the make file due to a bash requirement. Run\n# semgrep explicitly here. \nexport PATH=\"$HOME/.local/bin:$PATH\" \necho -n 'Semgrep Version: '\nsemgrep --version\nsemgrep --error --include '*.go' --exclude 'vendor' -f tools/semgrep/ci .\n" - name: Run Semgrep Rules - pre-flight-checks: - machine: - image: ubuntu-2004:2022.10.1 - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - steps: - - checkout - - run: - command: | - GO_VERSION=$(cat .go-version) - [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } - # Install Go - cd ~ - curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" - sudo rm -rf /usr/local/go - sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - rm -f "go${GO_VERSION}.linux-amd64.tar.gz" - GOPATH="/home/circleci/go" - mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } - mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; } - echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" - echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" - echo "export GOPROXY=https://proxy.golang.org,direct" >> "$BASH_ENV" - echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV" - - echo "$ go version" - go version - name: Setup Go - - run: - command: | - export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION - mkdir -p $CCI_PATH - NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH} - URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz - curl -sSL $URL \ - | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci" - # Add circleci to the path for subsequent steps. - echo "export PATH=$CCI_PATH:\$PATH" >> $BASH_ENV - # Done, print some debug info. - set -x - . $BASH_ENV - which circleci - circleci version - environment: - ARCH: linux_amd64 - BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download - name: Install CircleCI CLI - - run: - command: | - set -x - . $BASH_ENV - make ci-verify - name: Verify CircleCI - - add_ssh_keys: - fingerprints: - - b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9 - - run: - command: | - git config --global url."git@github.com:".insteadOf https://github.com/ - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}} - - v1.5-{{checksum "go.sum"}} - name: Restore closest matching go modules cache - - run: - command: | - # set GOPATH explicitly to download to the right cache - export GOPATH=$HOME/go - # go list ./... forces downloading some additional versions of modules that 'go mod - # download' misses. We need this because we make use of go list itself during - # code generation in later builds that rely on this module cache. - go list ./... - go mod download -json - ( cd sdk && go mod download -json; ) - ( cd api && go mod download -json; ) - name: go mod download - - run: - command: | - git --no-pager diff --exit-code || { - echo "ERROR: Files modified by go mod download, see above." - exit 1 - } - name: Verify downloading modules did not modify any files - - save_cache: - key: v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Save go modules cache - paths: - - /home/circleci/go/pkg/mod - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 - test-go-race-remote-docker: - docker: - - image: docker.mirror.hashicorp.services/cimg/go:1.20.1 - resource_class: medium - working_directory: /home/circleci/go/src/github.com/hashicorp/vault - parallelism: 8 - steps: - - run: - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 - name: Check branch name - working_directory: ~/ - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 20.10.17 - - add_ssh_keys: - fingerprints: - - b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9 - - run: - command: | - git config --global url."git@github.com:".insteadOf https://github.com/ - - run: - command: | - TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key - name: Compute test cache key - - restore_cache: - keys: - - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - - restore_cache: - keys: - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - name: Restore exact go modules cache - - run: - command: | - set -exo pipefail - - EXTRA_TAGS= - case "-race" in - *-race*) export VAULT_CI_GO_TEST_RACE=1;; - *) EXTRA_TAGS=deadlock;; - esac - - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - USE_DOCKER=0 - USE_DOCKER=1 - - # Check all directories with a go.mod file - modules=("." "api" "sdk") - all_package_names="" - - for dir in "${modules[@]}" - do - pushd "$dir" - # On its own line so that -e will fail the tests if we detect errors here. - go list -test -json ./... > test-list.json - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - # The appended true condition ensures the command will succeed if no packages are found - if [ $USE_DOCKER == 1 ]; then - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - else - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - fi - # Move back into root directory - popd - # Append the test packages into the global list, if any are found - if [ -n "$package_names" ]; then - all_package_names+=" ${package_names}" - fi - done - - # After running tests split step, we are now running the following steps - # in multiple different containers, each getting a different subset of - # the test packages in their package_names variable. Each container - # has its own remote docker VM. - - make prep - - mkdir -p test-results/go-test - - # We don't want VAULT_LICENSE set when running Go tests, because that's - # not what developers have in their environments and it could break some - # tests; it would be like setting VAULT_TOKEN. However some non-Go - # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we set VAULT_LICENSE in CI, and here we unset it. Instead of - # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want - # an externally supplied license can opt-in to using it. - export VAULT_LICENSE_CI="$VAULT_LICENSE" - VAULT_LICENSE= - - # Create a docker network for our test container - if [ $USE_DOCKER == 1 ]; then - # Despite the fact that we're using a circleci image (thus getting the - # version they chose for the docker cli) and that we're specifying a - # docker version to use for the remote docker instances, we occasionally - # see "client version too new, max supported version 1.39" errors for - # reasons unclear. - export DOCKER_API_VERSION=1.39 - - TEST_DOCKER_NETWORK_NAME="${CIRCLE_WORKFLOW_JOB_ID}-${CIRCLE_NODE_INDEX}" - export TEST_DOCKER_NETWORK_ID=$(docker network list --quiet --no-trunc --filter="name=${TEST_DOCKER_NETWORK_NAME}") - if [ -z $TEST_DOCKER_NETWORK_ID ]; then - docker network prune -f - TEST_DOCKER_NETWORK_ID=$(docker network create "${TEST_DOCKER_NETWORK_NAME}") - fi - - - - # Start a docker test container to run the tests in - CONTAINER_ID="$(docker run -d \ - -e TEST_DOCKER_NETWORK_ID \ - -e GOPRIVATE \ - -e DOCKER_CERT_PATH \ - -e DOCKER_HOST \ - -e DOCKER_MACHINE_NAME \ - -e DOCKER_TLS_VERIFY \ - -e NO_PROXY \ - -e VAULT_TEST_LOG_DIR=/tmp/testlogs \ - --network ${TEST_DOCKER_NETWORK_NAME} \ - $GO_IMAGE \ - tail -f /dev/null)" - mkdir workspace - echo ${CONTAINER_ID} > workspace/container_id - - # Hack: Docker permissions appear to have changed; let's explicitly - # add a new user/group with the correct host uid to the docker - # container, fixing all of these permissions issues correctly. We - # then have to run with this user consistently in the future. - # - # Notably, in this shell pipeline we see: - # uid=1001(circleci) gid=1002(circleci) groups=1002(circleci) - # - # but inside the docker image below, we see: - # uid=3434(circleci) gid=3434(circleci) groups=3434(circleci) - # - # See also: https://github.com/CircleCI-Public/cimg-base/issues/122 - export HOST_GID="$(id -g)" - export HOST_UID="$(id -u)" - export CONT_GID="$(docker exec ${CONTAINER_ID} sh -c 'id -g')" - export CONT_GNAME="$(docker exec ${CONTAINER_ID} sh -c 'id -g -n')" - export CONT_UID="$(docker exec ${CONTAINER_ID} sh -c 'id -u')" - if (( HOST_UID != CONT_UID )); then - # Only provision a group if necessary; otherwise reuse the - # existing one. - if (( HOST_GID != CONT_GID )); then - docker exec -e HOST_GID -e CONT_GNAME ${CONTAINER_ID} sh -c 'sudo groupmod -g $HOST_GID $CONT_GNAME' - fi - - docker exec -e CONT_GNAME -e HOST_UID ${CONTAINER_ID} sh -c 'sudo usermod -a -G $CONT_GNAME -u $HOST_UID circleci' - fi - - # Run tests - test -d /tmp/go-cache && docker cp /tmp/go-cache ${CONTAINER_ID}:/tmp/gocache - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault' - docker cp . ${CONTAINER_ID}:/home/circleci/go/src/github.com/hashicorp/vault/ - docker cp $DOCKER_CERT_PATH/ ${CONTAINER_ID}:$DOCKER_CERT_PATH - - # Copy the downloaded modules inside the container. - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/pkg' - docker cp "$(go env GOPATH)/pkg/mod" ${CONTAINER_ID}:/home/circleci/go/pkg/mod - - docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \ - -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \ - -e GOCACHE=/tmp/gocache \ - -e GO_TAGS \ - -e GOPROXY="off" \ - -e VAULT_LICENSE_CI \ - -e GOARCH=amd64 \ - ${CONTAINER_ID} \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - -race \ - ${all_package_names} - else - GOARCH=amd64 \ - GOCACHE=/tmp/go-cache \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - -race \ - ${all_package_names} - fi - environment: - GOPRIVATE: github.com/hashicorp/* - name: Run Go tests - no_output_timeout: 60m - - run: - command: | - docker cp $(cat workspace/container_id):/home/circleci/go/src/github.com/hashicorp/vault/test-results . - docker cp $(cat workspace/container_id):/tmp/gocache /tmp/go-cache - name: Copy test results - when: always - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: /tmp/testlogs - environment: - - CIRCLECI_CLI_VERSION: 0.1.5546 - - GO_IMAGE: docker.mirror.hashicorp.services/cimg/go:1.20.1 - - GO_TAGS: '' - - GOFUMPT_VERSION: 0.3.1 - - GOTESTSUM_VERSION: 0.5.2 -workflows: - ci: - jobs: - - pre-flight-checks - - fmt - - install-ui-dependencies: - requires: - - pre-flight-checks - - build-go-dev: - requires: - - pre-flight-checks - - test-ui: - requires: - - install-ui-dependencies - - build-go-dev - - test-go: - requires: - - pre-flight-checks - - test-go-remote-docker: - requires: - - pre-flight-checks - - test-go-race: - requires: - - pre-flight-checks - - test-go-race-remote-docker: - requires: - - pre-flight-checks - - semgrep: - requires: - - pre-flight-checks - version: 2 diff --git a/.circleci/config/@config.yml b/.circleci/config/@config.yml deleted file mode 100644 index 38fbc6831210..000000000000 --- a/.circleci/config/@config.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -version: 2.1 - -orbs: - slack: circleci/slack@3.2.0 diff --git a/.circleci/config/commands/@caches.yml b/.circleci/config/commands/@caches.yml deleted file mode 100644 index 7ce217f074d6..000000000000 --- a/.circleci/config/commands/@caches.yml +++ /dev/null @@ -1,59 +0,0 @@ -restore_yarn_cache: - steps: - - restore_cache: - name: Restore yarn cache - key: &YARN_LOCK_CACHE_KEY yarn-lock-v7-{{ checksum "ui/yarn.lock" }} -save_yarn_cache: - steps: - - save_cache: - name: Save yarn cache - key: *YARN_LOCK_CACHE_KEY - paths: - - ui/node_modules -# allows restoring go mod caches by incomplete prefix. This is useful when re-generating -# cache, but not when running builds and tests that require an exact match. -# TODO should we be including arch in cache key? -restore_go_mod_cache_permissive: - steps: - - restore_cache: - name: Restore closest matching go modules cache - keys: - - &gocachekey v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}} - - v1.5-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}} - - v1.5-{{checksum "go.sum"}} -restore_go_mod_cache: - steps: - - restore_cache: - name: Restore exact go modules cache - keys: - - *gocachekey -save_go_mod_cache: - steps: - - save_cache: - name: Save go modules cache - key: *gocachekey - paths: - - /home/circleci/go/pkg/mod -refresh_go_mod_cache: - steps: - - restore_go_mod_cache_permissive - - run: - name: go mod download - command: | - # set GOPATH explicitly to download to the right cache - export GOPATH=$HOME/go - # go list ./... forces downloading some additional versions of modules that 'go mod - # download' misses. We need this because we make use of go list itself during - # code generation in later builds that rely on this module cache. - go list ./... - go mod download -json - ( cd sdk && go mod download -json; ) - ( cd api && go mod download -json; ) - - run: - name: Verify downloading modules did not modify any files - command: | - git --no-pager diff --exit-code || { - echo "ERROR: Files modified by go mod download, see above." - exit 1 - } - - save_go_mod_cache diff --git a/.circleci/config/commands/configure-git.yml b/.circleci/config/commands/configure-git.yml deleted file mode 100644 index a725ab97e7b9..000000000000 --- a/.circleci/config/commands/configure-git.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: - - add_ssh_keys: - fingerprints: - # "CircleCI Additional SSH Key" associated with hc-github-team-secure-vault-core GitHub user - - "b8:e2:38:f8:5b:1b:82:f3:1f:23:fa:46:6e:95:e7:e9" - - run: | - git config --global url."git@github.com:".insteadOf https://github.com/ diff --git a/.circleci/config/commands/exit-if-branch-does-not-need-test-ui.yml b/.circleci/config/commands/exit-if-branch-does-not-need-test-ui.yml deleted file mode 100644 index 771ef4d925f8..000000000000 --- a/.circleci/config/commands/exit-if-branch-does-not-need-test-ui.yml +++ /dev/null @@ -1,17 +0,0 @@ -description: > - Check if branch name starts with ui/ or docs/ and if so, exit. -steps: - - run: - working_directory: ~/ - name: Check branch name - command: | - case "$CIRCLE_BRANCH" in - main|ui/*|backport/ui/*|release/*|merge*) ;; - *) # If the branch being tested doesn't match one of the above patterns, - # we don't need to run test-ui and can abort the job. - circleci-agent step halt - ;; - esac - - # exit with success either way - exit 0 diff --git a/.circleci/config/commands/exit-if-ui-or-docs-branch.yml b/.circleci/config/commands/exit-if-ui-or-docs-branch.yml deleted file mode 100644 index 322091f70ba7..000000000000 --- a/.circleci/config/commands/exit-if-ui-or-docs-branch.yml +++ /dev/null @@ -1,14 +0,0 @@ -description: > - Check if branch name starts with ui/ or docs/ and if so, exit. -steps: - - run: - working_directory: ~/ - name: Check branch name - command: | - # If the branch being tested starts with ui/ or docs/ we want to exit the job without failing - [[ "$CIRCLE_BRANCH" = ui/* || "$CIRCLE_BRANCH" = docs/* || "$CIRCLE_BRANCH" = backport/docs/* ]] && { - # stop the job from this step - circleci-agent step halt - } - # exit with success either way - exit 0 diff --git a/.circleci/config/commands/go_test.yml b/.circleci/config/commands/go_test.yml deleted file mode 100644 index 9e4b4daa9da4..000000000000 --- a/.circleci/config/commands/go_test.yml +++ /dev/null @@ -1,226 +0,0 @@ -description: run go tests -parameters: - extra_flags: - type: string - default: "" - log_dir: - type: string - default: "/tmp/testlogs" - cache_dir: - type: string - default: /tmp/go-cache - save_cache: - type: boolean - default: false - use_docker: - type: boolean - default: false - arch: - type: string - # Only supported for use_docker=false, and only other value allowed is 386 - default: amd64 # must be 386 or amd64 -steps: - - configure-git - - run: - name: Compute test cache key - command: | - TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key - - restore_cache: - keys: - - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - - restore_go_mod_cache - - run: - name: Run Go tests - no_output_timeout: 60m - environment: - GOPRIVATE: 'github.com/hashicorp/*' - command: | - set -exo pipefail - - EXTRA_TAGS= - case "<< parameters.extra_flags >>" in - *-race*) export VAULT_CI_GO_TEST_RACE=1;; - *) EXTRA_TAGS=deadlock;; - esac - - # Install CircleCI CLI - curl -sSL \ - "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ - | sudo tar --overwrite -xz \ - -C /usr/local/bin \ - "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" - - USE_DOCKER=0 - <<# parameters.use_docker >> - USE_DOCKER=1 - <> - - # Check all directories with a go.mod file - modules=("." "api" "sdk") - all_package_names="" - - for dir in "${modules[@]}" - do - pushd "$dir" - # On its own line so that -e will fail the tests if we detect errors here. - go list -test -json ./... > test-list.json - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - # The appended true condition ensures the command will succeed if no packages are found - if [ $USE_DOCKER == 1 ]; then - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - else - package_names=$(< test-list.json jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true) - fi - # Move back into root directory - popd - # Append the test packages into the global list, if any are found - if [ -n "$package_names" ]; then - all_package_names+=" ${package_names}" - fi - done - - # After running tests split step, we are now running the following steps - # in multiple different containers, each getting a different subset of - # the test packages in their package_names variable. Each container - # has its own remote docker VM. - - make prep - - mkdir -p test-results/go-test - - # We don't want VAULT_LICENSE set when running Go tests, because that's - # not what developers have in their environments and it could break some - # tests; it would be like setting VAULT_TOKEN. However some non-Go - # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we set VAULT_LICENSE in CI, and here we unset it. Instead of - # VAULT_LICENSE, we populate VAULT_LICENSE_CI, so that tests which want - # an externally supplied license can opt-in to using it. - export VAULT_LICENSE_CI="$VAULT_LICENSE" - VAULT_LICENSE= - - # Create a docker network for our test container - if [ $USE_DOCKER == 1 ]; then - # Despite the fact that we're using a circleci image (thus getting the - # version they chose for the docker cli) and that we're specifying a - # docker version to use for the remote docker instances, we occasionally - # see "client version too new, max supported version 1.39" errors for - # reasons unclear. - export DOCKER_API_VERSION=1.39 - - TEST_DOCKER_NETWORK_NAME="${CIRCLE_WORKFLOW_JOB_ID}-${CIRCLE_NODE_INDEX}" - export TEST_DOCKER_NETWORK_ID=$(docker network list --quiet --no-trunc --filter="name=${TEST_DOCKER_NETWORK_NAME}") - if [ -z $TEST_DOCKER_NETWORK_ID ]; then - docker network prune -f - TEST_DOCKER_NETWORK_ID=$(docker network create "${TEST_DOCKER_NETWORK_NAME}") - fi - - - - # Start a docker test container to run the tests in - CONTAINER_ID="$(docker run -d \ - -e TEST_DOCKER_NETWORK_ID \ - -e GOPRIVATE \ - -e DOCKER_CERT_PATH \ - -e DOCKER_HOST \ - -e DOCKER_MACHINE_NAME \ - -e DOCKER_TLS_VERIFY \ - -e NO_PROXY \ - -e VAULT_TEST_LOG_DIR=<< parameters.log_dir >> \ - --network ${TEST_DOCKER_NETWORK_NAME} \ - $GO_IMAGE \ - tail -f /dev/null)" - mkdir workspace - echo ${CONTAINER_ID} > workspace/container_id - - # Hack: Docker permissions appear to have changed; let's explicitly - # add a new user/group with the correct host uid to the docker - # container, fixing all of these permissions issues correctly. We - # then have to run with this user consistently in the future. - # - # Notably, in this shell pipeline we see: - # uid=1001(circleci) gid=1002(circleci) groups=1002(circleci) - # - # but inside the docker image below, we see: - # uid=3434(circleci) gid=3434(circleci) groups=3434(circleci) - # - # See also: https://github.com/CircleCI-Public/cimg-base/issues/122 - export HOST_GID="$(id -g)" - export HOST_UID="$(id -u)" - export CONT_GID="$(docker exec ${CONTAINER_ID} sh -c 'id -g')" - export CONT_GNAME="$(docker exec ${CONTAINER_ID} sh -c 'id -g -n')" - export CONT_UID="$(docker exec ${CONTAINER_ID} sh -c 'id -u')" - if (( HOST_UID != CONT_UID )); then - # Only provision a group if necessary; otherwise reuse the - # existing one. - if (( HOST_GID != CONT_GID )); then - docker exec -e HOST_GID -e CONT_GNAME ${CONTAINER_ID} sh -c 'sudo groupmod -g $HOST_GID $CONT_GNAME' - fi - - docker exec -e CONT_GNAME -e HOST_UID ${CONTAINER_ID} sh -c 'sudo usermod -a -G $CONT_GNAME -u $HOST_UID circleci' - fi - - # Run tests - test -d << parameters.cache_dir >> && docker cp << parameters.cache_dir >> ${CONTAINER_ID}:/tmp/gocache - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/src/github.com/hashicorp/vault' - docker cp . ${CONTAINER_ID}:/home/circleci/go/src/github.com/hashicorp/vault/ - docker cp $DOCKER_CERT_PATH/ ${CONTAINER_ID}:$DOCKER_CERT_PATH - - # Copy the downloaded modules inside the container. - docker exec ${CONTAINER_ID} sh -c 'mkdir -p /home/circleci/go/pkg' - docker cp "$(go env GOPATH)/pkg/mod" ${CONTAINER_ID}:/home/circleci/go/pkg/mod - - docker exec -w /home/circleci/go/src/github.com/hashicorp/vault/ \ - -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \ - -e GOCACHE=/tmp/gocache \ - -e GO_TAGS \ - -e GOPROXY="off" \ - -e VAULT_LICENSE_CI \ - -e GOARCH=<< parameters.arch >> \ - ${CONTAINER_ID} \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - << parameters.extra_flags >> \ - ${all_package_names} - else - GOARCH=<< parameters.arch >> \ - GOCACHE=<< parameters.cache_dir >> \ - gotestsum --format=short-verbose \ - --junitfile test-results/go-test/results.xml \ - --jsonfile test-results/go-test/results.json \ - -- \ - -tags "${GO_TAGS} ${EXTRA_TAGS}" \ - -timeout=60m \ - -parallel=20 \ - << parameters.extra_flags >> \ - ${all_package_names} - fi - - - when: - condition: << parameters.use_docker >> - steps: - - run: - name: Copy test results - when: always - command: | - docker cp $(cat workspace/container_id):/home/circleci/go/src/github.com/hashicorp/vault/test-results . - docker cp $(cat workspace/container_id):/tmp/gocache << parameters.cache_dir >> - - when: - condition: << parameters.save_cache >> - steps: - - save_cache: - when: always - key: go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }} - paths: - - << parameters.cache_dir >> diff --git a/.circleci/config/commands/setup-go.yml b/.circleci/config/commands/setup-go.yml deleted file mode 100644 index 5aec0087e9da..000000000000 --- a/.circleci/config/commands/setup-go.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -description: > - Ensure the right version of Go is installed and set GOPATH to $HOME/go. -parameters: - GOPROXY: - description: > - Set GOPROXY. By default this is set to "off" meaning you have to have all modules pre-downloaded. - type: string - default: "off" - GOPRIVATE: - description: Set GOPRIVATE, defaults to github.com/hashicorp/* - type: string - default: github.com/hashicorp/* -steps: - - run: - name: Setup Go - command: | - GO_VERSION=$(cat .go-version) - [ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; } - # Install Go - cd ~ - curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" - sudo rm -rf /usr/local/go - sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - rm -f "go${GO_VERSION}.linux-amd64.tar.gz" - GOPATH="/home/circleci/go" - mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; } - mkdir $GOPATH/bin 2>/dev/null || { sudo mkdir $GOPATH/bin && sudo chmod 777 $GOPATH/bin; } - echo "export GOPATH='$GOPATH'" >> "$BASH_ENV" - echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV" - echo "export GOPROXY=<>" >> "$BASH_ENV" - echo "export GOPRIVATE=<>" >> "$BASH_ENV" - - echo "$ go version" - go version diff --git a/.circleci/config/executors/@executors.yml b/.circleci/config/executors/@executors.yml deleted file mode 100644 index 586fd70e7363..000000000000 --- a/.circleci/config/executors/@executors.yml +++ /dev/null @@ -1,49 +0,0 @@ -references: - environment: &ENVIRONMENT - CIRCLECI_CLI_VERSION: 0.1.5546 # Pin CircleCI CLI to patch version (ex: 1.2.3) - GOTESTSUM_VERSION: 0.5.2 # Pin gotestsum to patch version (ex: 1.2.3) - GOFUMPT_VERSION: 0.3.1 # Pin gofumpt to patch version (ex: 1.2.3) - GO_TAGS: "" - GO_IMAGE: &GO_IMAGE "docker.mirror.hashicorp.services/cimg/go:1.20.1" -go-machine: - machine: - image: ubuntu-2004:2022.10.1 - environment: *ENVIRONMENT - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -node: - docker: - - image: docker.mirror.hashicorp.services/circleci/node:14-browsers - environment: - # See https://git.io/vdao3 for details. - JOBS: 2 - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -python: - docker: - - image: docker.mirror.hashicorp.services/python:3-alpine - shell: /usr/bin/env bash -euo pipefail -c - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -semgrep: - docker: - - image: docker.mirror.hashicorp.services/returntocorp/semgrep:0.113.0 - shell: /bin/sh - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -docker-env-go-test-remote-docker: - resource_class: medium - docker: - - image: *GO_IMAGE - environment: *ENVIRONMENT - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -docker-env-go-test: - resource_class: large - docker: - - image: *GO_IMAGE - environment: *ENVIRONMENT - working_directory: /home/circleci/go/src/github.com/hashicorp/vault -docker-env-go-test-race: - resource_class: xlarge - docker: - - image: *GO_IMAGE - environment: *ENVIRONMENT - working_directory: /home/circleci/go/src/github.com/hashicorp/vault diff --git a/.circleci/config/jobs/build-go-dev.yml b/.circleci/config/jobs/build-go-dev.yml deleted file mode 100644 index cce6d3f61edd..000000000000 --- a/.circleci/config/jobs/build-go-dev.yml +++ /dev/null @@ -1,20 +0,0 @@ -executor: go-machine -steps: - - checkout - - setup-go - - restore_go_mod_cache - - attach_workspace: - at: . - - run: - name: Build dev binary - command: | - # Move dev UI assets to expected location - rm -rf ./pkg - mkdir ./pkg - - # Build dev binary - make ci-bootstrap dev - - persist_to_workspace: - root: . - paths: - - bin diff --git a/.circleci/config/jobs/fmt.yml b/.circleci/config/jobs/fmt.yml deleted file mode 100644 index 7d9a08dcebd1..000000000000 --- a/.circleci/config/jobs/fmt.yml +++ /dev/null @@ -1,17 +0,0 @@ -description: Ensure go formatting is correct. -executor: go-machine -steps: - - checkout - # Setup Go enabling the proxy for downloading modules. - - setup-go: - GOPROXY: https://proxy.golang.org,direct - - run: - name: make fmt - command: | - echo "Using gofumpt version ${GOFUMPT_VERSION}" - go install "mvdan.cc/gofumpt@v${GOFUMPT_VERSION}" - make fmt - if ! git diff --exit-code; then - echo "Code has formatting errors. Run 'make fmt' to fix" - exit 1 - fi diff --git a/.circleci/config/jobs/install-ui-dependencies.yml b/.circleci/config/jobs/install-ui-dependencies.yml deleted file mode 100644 index 845e0c7770e2..000000000000 --- a/.circleci/config/jobs/install-ui-dependencies.yml +++ /dev/null @@ -1,11 +0,0 @@ -executor: node -steps: - - checkout - - restore_yarn_cache - - run: - name: Install UI dependencies - command: | - cd ui - yarn install - npm rebuild node-sass - - save_yarn_cache diff --git a/.circleci/config/jobs/pre-flight-checks.yml b/.circleci/config/jobs/pre-flight-checks.yml deleted file mode 100644 index 924b451b51d2..000000000000 --- a/.circleci/config/jobs/pre-flight-checks.yml +++ /dev/null @@ -1,34 +0,0 @@ -description: Ensure nothing obvious is broken, and pre-cache Go modules. -executor: go-machine -steps: - - checkout - # Setup Go enabling the proxy for downloading modules. - - setup-go: - GOPROXY: https://proxy.golang.org,direct - - run: - name: Install CircleCI CLI - environment: - ARCH: linux_amd64 - BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download - command: | - export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION - mkdir -p $CCI_PATH - NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH} - URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz - curl -sSL $URL \ - | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci" - # Add circleci to the path for subsequent steps. - echo "export PATH=$CCI_PATH:\$PATH" >> $BASH_ENV - # Done, print some debug info. - set -x - . $BASH_ENV - which circleci - circleci version - - run: - name: Verify CircleCI - command: | - set -x - . $BASH_ENV - make ci-verify - - configure-git - - refresh_go_mod_cache diff --git a/.circleci/config/jobs/semgrep.yml b/.circleci/config/jobs/semgrep.yml deleted file mode 100644 index c5cf749e129d..000000000000 --- a/.circleci/config/jobs/semgrep.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -executor: semgrep -steps: - - checkout - - attach_workspace: - at: . - - run: - name: Run Semgrep Rules - command: | - # Alpine images can't run the make file due to a bash requirement. Run - # semgrep explicitly here. - export PATH="$HOME/.local/bin:$PATH" - echo -n 'Semgrep Version: ' - semgrep --version - semgrep --error --include '*.go' --exclude 'vendor' -f tools/semgrep/ci . diff --git a/.circleci/config/jobs/test-go-nightly.yml b/.circleci/config/jobs/test-go-nightly.yml deleted file mode 100644 index 502cdfa4e185..000000000000 --- a/.circleci/config/jobs/test-go-nightly.yml +++ /dev/null @@ -1,14 +0,0 @@ -executor: go-machine -steps: - - checkout - - setup-go - - restore_go_mod_cache - - go_test: - log_dir: "/tmp/testlogs" - save_cache: true - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: "/tmp/testlogs" diff --git a/.circleci/config/jobs/test-go-race-remote-docker.yml b/.circleci/config/jobs/test-go-race-remote-docker.yml deleted file mode 100644 index 6780c60366e4..000000000000 --- a/.circleci/config/jobs/test-go-race-remote-docker.yml +++ /dev/null @@ -1,18 +0,0 @@ -executor: docker-env-go-test-remote-docker -parallelism: 8 -steps: - - exit-if-ui-or-docs-branch - - checkout - - setup_remote_docker: - version: 20.10.17 - docker_layer_caching: true - - go_test: - extra_flags: "-race" - log_dir: "/tmp/testlogs" - use_docker: true - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: "/tmp/testlogs" diff --git a/.circleci/config/jobs/test-go-race.yml b/.circleci/config/jobs/test-go-race.yml deleted file mode 100644 index fcda05e9ceda..000000000000 --- a/.circleci/config/jobs/test-go-race.yml +++ /dev/null @@ -1,14 +0,0 @@ -executor: docker-env-go-test-race -parallelism: 8 -steps: - - exit-if-ui-or-docs-branch - - checkout - - go_test: - extra_flags: "-race" - log_dir: "/tmp/testlogs" - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: "/tmp/testlogs" diff --git a/.circleci/config/jobs/test-go-remote-docker.yml b/.circleci/config/jobs/test-go-remote-docker.yml deleted file mode 100644 index f51003f09445..000000000000 --- a/.circleci/config/jobs/test-go-remote-docker.yml +++ /dev/null @@ -1,17 +0,0 @@ -executor: docker-env-go-test-remote-docker -parallelism: 8 -steps: - - exit-if-ui-or-docs-branch - - checkout - - setup_remote_docker: - version: 20.10.17 - docker_layer_caching: true - - go_test: - log_dir: "/tmp/testlogs" - use_docker: true - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: "/tmp/testlogs" diff --git a/.circleci/config/jobs/test-go.yml b/.circleci/config/jobs/test-go.yml deleted file mode 100644 index c1674de870d2..000000000000 --- a/.circleci/config/jobs/test-go.yml +++ /dev/null @@ -1,13 +0,0 @@ -executor: docker-env-go-test -parallelism: 8 -steps: - - exit-if-ui-or-docs-branch - - checkout - - go_test: - log_dir: "/tmp/testlogs" - - store_artifacts: - path: test-results - - store_test_results: - path: test-results - - store_artifacts: - path: "/tmp/testlogs" diff --git a/.circleci/config/jobs/test-ui.yml b/.circleci/config/jobs/test-ui.yml deleted file mode 100644 index f2aa19b0508d..000000000000 --- a/.circleci/config/jobs/test-ui.yml +++ /dev/null @@ -1,22 +0,0 @@ -executor: node -resource_class: xlarge -steps: - - exit-if-branch-does-not-need-test-ui - - checkout - - restore_yarn_cache - - attach_workspace: - at: . - - run: - name: Test UI - command: | - # Add ./bin to the PATH so vault binary can be run by Ember tests - export PATH="${PWD}/bin:${PATH}" - - # Run Ember tests - cd ui - mkdir -p test-results/qunit - yarn test:oss - - store_artifacts: - path: ui/test-results - - store_test_results: - path: ui/test-results diff --git a/.circleci/config/workflows/ci.yml b/.circleci/config/workflows/ci.yml deleted file mode 100644 index 5e99293d7ea3..000000000000 --- a/.circleci/config/workflows/ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -jobs: - - pre-flight-checks - - fmt - - install-ui-dependencies: - requires: - - pre-flight-checks - - build-go-dev: - requires: - - pre-flight-checks - - test-ui: - requires: - - install-ui-dependencies - - build-go-dev - # Only main, UI, release and merge branches need to run UI tests. - # We don't filter here however because test-ui is configured in github as - # required so it must run, instead we short-circuit within test-ui. - - test-go: - requires: - - pre-flight-checks - # We don't filter here because this is a required CI check; - # instead we short-circuit within the test command so it ends quickly. - - test-go-remote-docker: - requires: - - pre-flight-checks - # We don't filter here because this is a required CI check; - # instead we short-circuit within the test command so it ends quickly. - - test-go-race: - requires: - - pre-flight-checks - - test-go-race-remote-docker: - requires: - - pre-flight-checks - - semgrep: - requires: - - pre-flight-checks diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000000..335ce6d5aab0 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,10 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - small + - large + - ondemand + diff --git a/.github/scripts/generate-test-package-lists.sh b/.github/scripts/generate-test-package-lists.sh new file mode 100755 index 000000000000..493a92c8c87c --- /dev/null +++ b/.github/scripts/generate-test-package-lists.sh @@ -0,0 +1,283 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This script is meant to be sourced into the shell running in a Github +# workflow. + +# This script is a temporary measure until we implement a dynamic test-splitting +# solution. It distributes the entire set of test packages into 16 sublists, +# which should roughly take an equal amount of time to complete. + +test_packages=() + +base="github.com/hashicorp/vault" + +# Total time: 526 +test_packages[1]+=" $base/api" +test_packages[1]+=" $base/command" +test_packages[1]+=" $base/sdk/helper/keysutil" + +# Total time: 1160 +test_packages[2]+=" $base/sdk/helper/ocsp" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[2]+=" $base/vault/external_tests/replication-perf" +fi + +# Total time: 1009 +test_packages[3]+=" $base/builtin/credential/approle" +test_packages[3]+=" $base/command/agent/sink/file" +test_packages[3]+=" $base/command/agent/template" +test_packages[3]+=" $base/helper/random" +test_packages[3]+=" $base/helper/storagepacker" +test_packages[3]+=" $base/sdk/helper/certutil" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[3]+=" $base/vault/external_tests/entropy" +fi +test_packages[3]+=" $base/vault/external_tests/raft" + +# Total time: 830 +test_packages[4]+=" $base/builtin/plugin" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[4]+=" $base/enthelpers/fsm" +fi +test_packages[4]+=" $base/http" +test_packages[4]+=" $base/sdk/helper/pluginutil" +test_packages[4]+=" $base/serviceregistration/kubernetes" +test_packages[4]+=" $base/tools/godoctests/pkg/analyzer" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[4]+=" $base/vault/external_tests/apilock" + test_packages[4]+=" $base/vault/external_tests/filteredpaths" + test_packages[4]+=" $base/vault/external_tests/perfstandby" + test_packages[4]+=" $base/vault/external_tests/replication-dr" +fi + + +# Total time: 258 +test_packages[5]+=" $base/builtin/credential/aws" +test_packages[5]+=" $base/builtin/credential/cert" +test_packages[5]+=" $base/builtin/logical/aws" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[5]+=" $base/enthelpers/logshipper" + test_packages[5]+=" $base/enthelpers/merkle" +fi +test_packages[5]+=" $base/helper/hostutil" +test_packages[5]+=" $base/helper/pgpkeys" +test_packages[5]+=" $base/sdk/physical/inmem" +test_packages[5]+=" $base/vault/activity" +test_packages[5]+=" $base/vault/diagnose" +test_packages[5]+=" $base/vault/external_tests/pprof" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[5]+=" $base/vault/external_tests/resolver" +fi +test_packages[5]+=" $base/vault/external_tests/response" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[5]+=" $base/vault/external_tests/seal" +fi +test_packages[5]+=" $base/vault/external_tests/sealmigration" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[5]+=" $base/vault/external_tests/transform" +fi + +# Total time: 588 +test_packages[6]+=" $base" +test_packages[6]+=" $base/audit" +test_packages[6]+=" $base/builtin/audit/file" +test_packages[6]+=" $base/builtin/credential/github" +test_packages[6]+=" $base/builtin/credential/okta" +test_packages[6]+=" $base/builtin/logical/database/dbplugin" +test_packages[6]+=" $base/command/agent/auth/cert" +test_packages[6]+=" $base/command/agent/auth/jwt" +test_packages[6]+=" $base/command/agent/auth/kerberos" +test_packages[6]+=" $base/command/agent/auth/kubernetes" +test_packages[6]+=" $base/command/agent/auth/token-file" +test_packages[6]+=" $base/command/agent/cache" +test_packages[6]+=" $base/command/agent/cache/cacheboltdb" +test_packages[6]+=" $base/command/agent/cache/cachememdb" +test_packages[6]+=" $base/command/agent/cache/keymanager" +test_packages[6]+=" $base/command/agent/config" +test_packages[6]+=" $base/command/config" +test_packages[6]+=" $base/command/token" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[6]+=" $base/enthelpers/namespace" + test_packages[6]+=" $base/enthelpers/replicatedpaths" + test_packages[6]+=" $base/enthelpers/sealrewrap" +fi +test_packages[6]+=" $base/helper/builtinplugins" +test_packages[6]+=" $base/helper/dhutil" +test_packages[6]+=" $base/helper/fairshare" +test_packages[6]+=" $base/helper/flag-kv" +test_packages[6]+=" $base/helper/flag-slice" +test_packages[6]+=" $base/helper/forwarding" +test_packages[6]+=" $base/helper/logging" +test_packages[6]+=" $base/helper/metricsutil" +test_packages[6]+=" $base/helper/namespace" +test_packages[6]+=" $base/helper/osutil" +test_packages[6]+=" $base/helper/parseip" +test_packages[6]+=" $base/helper/policies" +test_packages[6]+=" $base/helper/testhelpers/logical" +test_packages[6]+=" $base/helper/timeutil" +test_packages[6]+=" $base/helper/useragent" +test_packages[6]+=" $base/helper/versions" +test_packages[6]+=" $base/internalshared/configutil" +test_packages[6]+=" $base/internalshared/listenerutil" +test_packages[6]+=" $base/physical/alicloudoss" +test_packages[6]+=" $base/physical/gcs" +test_packages[6]+=" $base/physical/manta" +test_packages[6]+=" $base/physical/mssql" +test_packages[6]+=" $base/physical/oci" +test_packages[6]+=" $base/physical/s3" +test_packages[6]+=" $base/physical/spanner" +test_packages[6]+=" $base/physical/swift" +test_packages[6]+=" $base/physical/zookeeper" +test_packages[6]+=" $base/plugins/database/hana" +test_packages[6]+=" $base/plugins/database/redshift" +test_packages[6]+=" $base/sdk/database/dbplugin/v5" +test_packages[6]+=" $base/sdk/database/helper/credsutil" +test_packages[6]+=" $base/sdk/helper/authmetadata" +test_packages[6]+=" $base/sdk/helper/compressutil" +test_packages[6]+=" $base/sdk/helper/cryptoutil" +test_packages[6]+=" $base/sdk/helper/identitytpl" +test_packages[6]+=" $base/sdk/helper/kdf" +test_packages[6]+=" $base/sdk/helper/locksutil" +test_packages[6]+=" $base/sdk/helper/pathmanager" +test_packages[6]+=" $base/sdk/helper/roottoken" +test_packages[6]+=" $base/sdk/helper/testhelpers/schema" +test_packages[6]+=" $base/sdk/helper/xor" +test_packages[6]+=" $base/sdk/physical/file" +test_packages[6]+=" $base/sdk/plugin/pb" +test_packages[6]+=" $base/serviceregistration/kubernetes/client" +test_packages[6]+=" $base/shamir" +test_packages[6]+=" $base/vault/cluster" +test_packages[6]+=" $base/vault/eventbus" +test_packages[6]+=" $base/vault/external_tests/api" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[6]+=" $base/vault/external_tests/consistencyheaders" +fi +test_packages[6]+=" $base/vault/external_tests/expiration" +test_packages[6]+=" $base/vault/external_tests/hcp_link" +test_packages[6]+=" $base/vault/external_tests/kv" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[6]+=" $base/vault/external_tests/plugins" +fi +test_packages[6]+=" $base/vault/external_tests/quotas" +test_packages[6]+=" $base/vault/seal" + +# Total time: 389 +test_packages[7]+=" $base/builtin/credential/userpass" +test_packages[7]+=" $base/builtin/logical/pki" +test_packages[7]+=" $base/builtin/logical/transit" +test_packages[7]+=" $base/command/agent" +test_packages[7]+=" $base/helper/monitor" +test_packages[7]+=" $base/sdk/database/helper/connutil" +test_packages[7]+=" $base/sdk/database/helper/dbutil" +test_packages[7]+=" $base/sdk/helper/cidrutil" +test_packages[7]+=" $base/sdk/helper/custommetadata" +test_packages[7]+=" $base/sdk/helper/jsonutil" +test_packages[7]+=" $base/sdk/helper/ldaputil" +test_packages[7]+=" $base/sdk/helper/logging" +test_packages[7]+=" $base/sdk/helper/policyutil" +test_packages[7]+=" $base/sdk/helper/salt" +test_packages[7]+=" $base/sdk/helper/template" +test_packages[7]+=" $base/sdk/helper/useragent" +test_packages[7]+=" $base/sdk/logical" +test_packages[7]+=" $base/sdk/plugin/mock" +test_packages[7]+=" $base/sdk/queue" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[7]+=" $base/vault/autosnapshots" + test_packages[7]+=" $base/vault/external_tests/activity" +fi +test_packages[7]+=" $base/vault/external_tests/approle" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[7]+=" $base/vault/external_tests/kmip" +fi +test_packages[7]+=" $base/vault/external_tests/mfa" +test_packages[7]+=" $base/vault/external_tests/misc" +test_packages[7]+=" $base/vault/quotas" + +# Total time: 779 +test_packages[8]+=" $base/builtin/credential/aws/pkcs7" +test_packages[8]+=" $base/builtin/logical/totp" +test_packages[8]+=" $base/command/agent/auth" +test_packages[8]+=" $base/physical/raft" +test_packages[8]+=" $base/sdk/framework" +test_packages[8]+=" $base/sdk/plugin" +test_packages[8]+=" $base/vault" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[8]+=" $base/vault/external_tests/barrier" + test_packages[8]+=" $base/vault/external_tests/cubbyholes" +fi +test_packages[8]+=" $base/vault/external_tests/metrics" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[8]+=" $base/vault/external_tests/replication" +fi +test_packages[8]+=" $base/vault/external_tests/router" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[8]+=" $base/vault/external_tests/system" + test_packages[8]+=" $base/vault/managed_key" +fi + +# Total time: 310 +test_packages[9]+=" $base/vault/hcp_link/capabilities/api_capability" +test_packages[9]+=" $base/vault/external_tests/plugin" + +# Total time: 925 +test_packages[10]+=" $base/builtin/credential/ldap" +test_packages[10]+=" $base/builtin/logical/database" +test_packages[10]+=" $base/physical/etcd" +test_packages[10]+=" $base/physical/postgresql" + +# Total time: 851 +test_packages[11]+=" $base/builtin/logical/rabbitmq" +test_packages[11]+=" $base/physical/dynamodb" +test_packages[11]+=" $base/plugins/database/influxdb" +test_packages[11]+=" $base/vault/external_tests/identity" +test_packages[11]+=" $base/vault/external_tests/token" + +# Total time: 340 +test_packages[12]+=" $base/builtin/logical/consul" +test_packages[12]+=" $base/physical/couchdb" +test_packages[12]+=" $base/plugins/database/mongodb" +test_packages[12]+=" $base/plugins/database/mssql" +test_packages[12]+=" $base/plugins/database/mysql" + +# Total time: 704 +test_packages[13]+=" $base/builtin/logical/pkiext" +test_packages[13]+=" $base/command/server" +test_packages[13]+=" $base/physical/aerospike" +test_packages[13]+=" $base/physical/cockroachdb" +test_packages[13]+=" $base/plugins/database/postgresql" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[13]+=" $base/vault/external_tests/filteredpathsext" +fi +test_packages[13]+=" $base/vault/external_tests/policy" + +# Total time: 374 +test_packages[14]+=" $base/builtin/credential/radius" +test_packages[14]+=" $base/builtin/logical/ssh" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[14]+=" $base/enthelpers/wal" +fi +test_packages[14]+=" $base/physical/azure" +test_packages[14]+=" $base/serviceregistration/consul" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[14]+=" $base/vault/external_tests/quotas-docker" +fi +test_packages[14]+=" $base/vault/external_tests/raftha" + +# Total time: 362 +test_packages[15]+=" $base/builtin/logical/nomad" +test_packages[15]+=" $base/physical/mysql" +test_packages[15]+=" $base/plugins/database/cassandra" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[15]+=" $base/vault/external_tests/namespaces" +fi +test_packages[15]+=" $base/vault/external_tests/sealmigrationext" + +# Total time: 635 +test_packages[16]+=" $base/physical/cassandra" +test_packages[16]+=" $base/physical/consul" +if [ "${ENTERPRISE:+x}" == "x" ] ; then + test_packages[16]+=" $base/vault/external_tests/autosnapshots" + test_packages[16]+=" $base/vault/external_tests/replicationext" + test_packages[16]+=" $base/vault/external_tests/sealext" +fi diff --git a/.github/scripts/test-generate-test-package-lists.sh b/.github/scripts/test-generate-test-package-lists.sh new file mode 100755 index 000000000000..c3d1cb60670b --- /dev/null +++ b/.github/scripts/test-generate-test-package-lists.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +set -e${DEBUG+x}o pipefail + +# +# This script is run to make sure that every package returned by +# go list -test ./... (when run from the repo root, api/, and sdk/ directory) +# appear in the test_packages array defined in the sibling file +# generate-test-package-lists.sh +# +# This script is executed as part of the ci workflow triggered by pull_requests +# events. In the event that the job that runs this script fails, examine the +# output of the 'test' step in that job to obtain the list of test packages that +# are missing in the test_packages array or that should be removed from it. +# + +dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source generate-test-package-lists.sh + +get_module_packages() { + local package_list=($(go list -test -json ./... | jq -r '.ForTest | select(.!=null)' | grep -v vault/integ | grep '^github.com/hashicorp/')) + + for package in "${package_list[@]}" ; do + # Check if the current package already exists in all_packages + if ! grep "\b$package\b" <<< "${all_packages[@]}" &> /dev/null; then + all_packages+=($package) + fi + done +} + +find_packages() { + for package in "${all_packages[@]}" ; do + if ! grep "\b${package}\b" <<< "${test_packages[@]}" &> /dev/null ; then + echo "Error: package ${package} is not present in test_packages" + exit 1 + fi + done +} + +count_test_packages() { + count=0 + for test_package in "${test_packages[@]}" ; do + count=$((${count}+$(wc -w <<< "${test_package}"))) + done + + echo $count +} + +all_packages=() + +cd "$dir/../.." +get_module_packages + +cd "$dir/../../sdk" +get_module_packages + +cd "$dir/../../api" +get_module_packages + +find_packages + +test_package_count=$(count_test_packages) +if (( ${#all_packages[@]} != $test_package_count )) ; then + echo "Error: there are currently ${#all_packages[@]} packages in the repository but $test_package_count packages in test_packages" + + unused_packages="${test_packages[@]} " + for ap in ${all_packages[@]} ; do + unused_packages="$(echo "$unused_packages" | sed -r "s~$ap ~ ~" )" + done + + echo "Packages in test_packages that aren't used: ${unused_packages// /}" +fi diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 000000000000..abe7e7237b81 --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,15 @@ + +name: Lint GitHub Actions Workflows +on: + push: + pull_request: + paths: + - '.github/**' + +jobs: + actionlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - name: "Check workflow files" + uses: docker://docker.mirror.hashicorp.services/rhysd/actionlint@sha256:93834930f56ca380be3e9a3377670d7aa5921be251b9c774891a39b3629b83b8 diff --git a/.github/workflows/build-vault-oss.yml b/.github/workflows/build-vault-oss.yml index c7d8dc1e6d7a..0ba21db59999 100644 --- a/.github/workflows/build-vault-oss.yml +++ b/.github/workflows/build-vault-oss.yml @@ -63,7 +63,7 @@ jobs: env: GOARCH: ${{ inputs.goarch }} GOOS: ${{ inputs.goos }} - run: echo "ARTIFACT_BASENAME=$(make ci-get-artifact-basename)" >> $GITHUB_ENV + run: echo "ARTIFACT_BASENAME=$(make ci-get-artifact-basename)" >> "$GITHUB_ENV" - name: Bundle Vault env: BUNDLE_PATH: out/${{ env.ARTIFACT_BASENAME }}.zip @@ -93,8 +93,8 @@ jobs: - if: ${{ inputs.create-packages }} name: Determine package file names run: | - echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV - echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV + echo "RPM_PACKAGE=$(basename out/*.rpm)" >> "$GITHUB_ENV" + echo "DEB_PACKAGE=$(basename out/*.deb)" >> "$GITHUB_ENV" - if: ${{ inputs.create-packages }} uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0561661b0969..247a8d2abfe3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,13 +31,14 @@ jobs: # enos-run-matrices. MATRIX_MAX_TEST_GROUPS: 5 run: | - echo "build-date=$(make ci-get-date)" >> $GITHUB_OUTPUT - echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT - echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> $GITHUB_OUTPUT - echo "package-name=vault" >> $GITHUB_OUTPUT - echo "vault-base-version=$(make ci-get-version-base)" >> $GITHUB_OUTPUT - echo "vault-revision=$(make ci-get-revision)" >> $GITHUB_OUTPUT - echo "vault-version=$(make ci-get-version)" >> $GITHUB_OUTPUT + # shellcheck disable=SC2129 + echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT" + echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT" + echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT" + echo "package-name=vault" >> "$GITHUB_OUTPUT" + echo "vault-base-version=$(make ci-get-version-base)" >> "$GITHUB_OUTPUT" + echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT" + echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT" - uses: hashicorp/actions-generate-metadata@v1 id: generate-metadata-file with: diff --git a/.github/workflows/changelog-checker.yml b/.github/workflows/changelog-checker.yml index d8a380270b26..3811a767fb18 100644 --- a/.github/workflows/changelog-checker.yml +++ b/.github/workflows/changelog-checker.yml @@ -56,9 +56,9 @@ jobs: # Else, we found some toolchain files. Let's make sure the contents are correct. if ! grep -q 'release-note:change' "$toolchain_files" || ! grep -q '^core: Bump Go version to' "$toolchain_files"; then echo "Invalid format for changelog. Expected format:" - echo "```release-note:change" + echo '```release-note:change' echo "core: Bump Go version to x.y.z." - echo "```" + echo '```' exit 1 else echo "Found Go toolchain changelog entry in PR!" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..7f1874070719 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,266 @@ +name: CI +on: + pull_request: + push: + branches: + - "main" + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-standard: ${{ steps.setup-outputs.outputs.compute-standard }} + compute-larger: ${{ steps.setup-outputs.outputs.compute-larger }} + enterprise: ${{ steps.setup-outputs.outputs.enterprise }} + go-build-tags: ${{ steps.setup-outputs.outputs.go-build-tags }} + steps: + - id: setup-outputs + name: Setup outputs + run: | + github_repository="${{ github.repository }}" + + if [ "${github_repository##*/}" == "vault-enterprise" ] ; then + # shellcheck disable=SC2129 + echo 'compute-standard=["self-hosted","ondemand","linux","type=m5.xlarge"]' >> "$GITHUB_OUTPUT" + echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> "$GITHUB_OUTPUT" + echo 'enterprise=1' >> "$GITHUB_OUTPUT" + echo 'go-build-tags=ent,enterprise' >> "$GITHUB_OUTPUT" + else + # shellcheck disable=SC2129 + echo 'compute-standard=["custom", "linux", "small"]' >> "$GITHUB_OUTPUT" + echo 'compute-larger=["custom", "linux", "medium"]' >> "$GITHUB_OUTPUT" + echo 'enterprise=' >> "$GITHUB_OUTPUT" + echo 'go-build-tags=' >> "$GITHUB_OUTPUT" + fi + semgrep: + name: Semgrep + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }} + container: + image: returntocorp/semgrep@sha256:ffc6f3567654f9431456d49fd059dfe548f007c494a7eb6cd5a1a3e50d813fb3 + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - name: Run Semgrep Rules + id: semgrep + run: semgrep ci --include '*.go' --config 'tools/semgrep/ci' + setup-go-cache: + name: Go Caches + needs: + - setup + uses: ./.github/workflows/setup-go-cache.yml + with: + runs-on: ${{ needs.setup.outputs.compute-larger }} + secrets: inherit + fmt: + name: Check Format + needs: + - setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 + with: + go-version-file: ./.go-version + cache: true + - id: format + run: | + echo "Using gofumpt version $(go run mvdan.cc/gofumpt -version)" + make fmt + if ! git diff --exit-code; then + echo "Code has formatting errors. Run 'make fmt' to fix" + exit 1 + fi + diff-oss-ci: + name: Diff OSS + needs: + - setup + if: ${{ needs.setup.outputs.enterprise != '' && github.base_ref != '' }} + runs-on: ['self-hosted','ondemand','small'] + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + with: + fetch-depth: 0 + - id: determine-branch + run: | + branch="${{ github.base_ref }}" + + if [[ $branch = release/* ]] ; then + branch=${branch%%+ent} + + # Add OSS remote + git config --global user.email "github-team-secret-vault-core@hashicorp.com" + git config --global user.name "hc-github-team-secret-vault-core" + git remote add oss https://github.com/hashicorp/vault.git + git fetch oss "$branch" + + branch="oss/$branch" + else + branch="origin/$branch" + fi + + echo "BRANCH=$branch" >> "$GITHUB_OUTPUT" + - id: diff + run: | + ./.github/scripts/oss-diff.sh ${{ steps.determine-branch.outputs.BRANCH }} HEAD + test-go: + name: Run Go tests + needs: + - setup + - setup-go-cache + # Don't run this job for branches starting with 'ui/', 'docs/', or 'backport/docs/' + if: ${{ ! (startsWith( github.ref_name, 'ui/' ) || startsWith( github.ref_name, 'docs/' ) || startsWith( github.ref_name, 'backport/docs/') ) }} + uses: ./.github/workflows/test-go.yml + with: + name: '' + total-runners: 16 + go-arch: amd64 + go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock' + runs-on: ${{ needs.setup.outputs.compute-standard }} + enterprise: ${{ needs.setup.outputs.enterprise }} + secrets: inherit + test-go-race: + name: Run Go tests with data race detection + needs: + - setup + - setup-go-cache + # Don't run this job for branches starting with 'ui/', 'docs/', or 'backport/docs/' + if: ${{ ! (startsWith( github.ref_name, 'ui/' ) || startsWith( github.ref_name, 'docs/' ) || startsWith( github.ref_name, 'backport/docs/') ) }} + uses: ./.github/workflows/test-go.yml + with: + name: race + total-runners: 16 + env-vars: | + { + "VAULT_CI_GO_TEST_RACE": 1 + } + extra-flags: '-race' + go-arch: amd64 + go-build-tags: ${{ needs.setup.outputs.go-build-tags }} + runs-on: ${{ needs.setup.outputs.compute-larger }} + enterprise: ${{ needs.setup.outputs.enterprise }} + secrets: inherit + test-go-fips: + name: Run Go tests with FIPS configuration + # Only run this job for the enterprise repo if the branch doesn't start with 'ui/', 'docs/', or 'backport/docs/'. + if: | + needs.setup.outputs.enterprise == 1 && + ! (startsWith( github.ref_name, 'ui/' ) || startsWith( github.ref_name, 'docs/' ) || startsWith( github.ref_name, 'backport/docs/')) + needs: + - setup + - setup-go-cache + uses: ./.github/workflows/test-go.yml + with: + name: fips + total-runners: 16 + env-vars: | + { + "GOEXPERIMENT": "boringcrypto" + } + go-arch: amd64 + go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2' + runs-on: ${{ needs.setup.outputs.compute-standard }} + enterprise: ${{ needs.setup.outputs.enterprise }} + secrets: inherit + test-ui: + name: Test UI + # The test-ui Job is only run for pushes to main, ui/*, backport/ui/*, release/*, and merge* + if: github.ref_name == 'main' || startsWith(github.ref_name, 'ui/') || startsWith(github.ref_name, 'backport/ui/') || startsWith(github.ref_name, 'release/') || startsWith(github.ref_name, 'merge') + needs: + - setup + permissions: + id-token: write + contents: read + runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 + with: + go-version-file: ./.go-version + cache: true + # Setup node.js without caching to allow running npm install -g yarn (next step) + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c + with: + node-version: 14 + - id: install-yarn + run: | + npm install -g yarn + # Setup node.js with caching using the yarn.lock file + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c + with: + node-version: 14 + cache: yarn + cache-dependency-path: ui/yarn.lock + - id: install-browser-libraries + run: sudo apt install -y libnss3-dev libgdk-pixbuf2.0-dev libgtk-3-dev libxss-dev libasound2 + - id: install-browser + uses: browser-actions/setup-chrome@597130847c84cdac5acceccbd676d612e6f8beb8 + - id: ui-dependencies + name: ui-dependencies + working-directory: ./ui + run: | + yarn install --frozen-lockfile + npm rebuild node-sass + - id: vault-auth + name: Authenticate to Vault + if: github.repository == 'hashicorp/vault-enterprise' + run: vault-auth + - id: secrets + name: Fetch secrets + if: github.repository == 'hashicorp/vault-enterprise' + uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/hashicorp/vault-enterprise/github-token token | PRIVATE_REPO_GITHUB_TOKEN; + kv/data/github/hashicorp/vault-enterprise/license license_1 | VAULT_LICENSE; + - id: setup-git + name: Setup Git + if: github.repository == 'hashicorp/vault-enterprise' + env: + PRIVATE_REPO_GITHUB_TOKEN: ${{ steps.secrets.outputs.PRIVATE_REPO_GITHUB_TOKEN }} + run: | + git config --global url."https://hc-github-team-secure-vault-core:${PRIVATE_REPO_GITHUB_TOKEN}@github.com".insteadOf https://github.com + - id: build-go-dev + name: build-go-dev + run: | + rm -rf ./pkg + mkdir ./pkg + + make ci-bootstrap dev + - id: test-ui + name: test-ui + env: + VAULT_LICENSE: ${{ steps.secrets.outputs.VAULT_LICENSE }} + run: | + export PATH="${PWD}/bin:${PATH}" + + if [ "${{ github.repository }}" == 'hashicorp/vault' ] ; then + export VAULT_LICENSE="${{ secrets.VAULT_LICENSE }}" + fi + + # Run Ember tests + cd ui + mkdir -p test-results/qunit + yarn test:oss + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-results-ui + path: ui/test-results + if: always() + - uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f + with: + paths: "ui/test-results/qunit/results.xml" + show: "fail" + if: always() + tests-completed: + needs: + - setup + - test-go + - test-ui + runs-on: ${{ fromJSON(needs.setup.outputs.compute-standard) }} + steps: + - run: echo "All Go test successfully passed" diff --git a/.github/workflows/enos-release-testing-oss.yml b/.github/workflows/enos-release-testing-oss.yml index 5fe50e8da061..f62089d994f1 100644 --- a/.github/workflows/enos-release-testing-oss.yml +++ b/.github/workflows/enos-release-testing-oss.yml @@ -27,9 +27,10 @@ jobs: # enos-run-matrices. MATRIX_MAX_TEST_GROUPS: 2 run: | - echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> $GITHUB_OUTPUT - echo "vault-revision=$(make ci-get-revision)" >> $GITHUB_OUTPUT - echo "vault-version=$(make ci-get-version)" >> $GITHUB_OUTPUT + # shellcheck disable=SC2129 + echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT" + echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT" + echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT" test: name: Test ${{ matrix.build-artifact-name }} diff --git a/.github/workflows/enos-run-k8s.yml b/.github/workflows/enos-run-k8s.yml index e306966c1abe..e5200d025151 100644 --- a/.github/workflows/enos-run-k8s.yml +++ b/.github/workflows/enos-run-k8s.yml @@ -44,7 +44,7 @@ jobs: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - name: Download Docker Image id: download - uses: actions/download-artifact@v3 + uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b with: name: ${{ inputs.artifact-name }} path: ./enos/support/downloads @@ -53,16 +53,16 @@ jobs: IS_ENT: ${{ startsWith(env.ARTIFACT_NAME, 'vault-enterprise' ) }} run: | mkdir -p ./enos/support/terraform-plugin-cache - if ${IS_ENT} == true; then + if [ "$IS_ENT" == true ]; then echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true - echo "edition=ent" >> $GITHUB_ENV + echo "edition=ent" >> "$GITHUB_ENV" echo "edition set to 'ent'" - echo "image_repo=hashicorp/vault-enterprise" >> $GITHUB_ENV + echo "image_repo=hashicorp/vault-enterprise" >> "$GITHUB_ENV" echo "image repo set to 'hashicorp/vault-enterprise'" else - echo "edition=oss" >> $GITHUB_ENV + echo "edition=oss" >> "$GITHUB_ENV" echo "edition set to 'oss'" - echo "image_repo=hashicorp/vault" >> $GITHUB_ENV + echo "image_repo=hashicorp/vault" >> "$GITHUB_ENV" echo "image repo set to 'hashicorp/vault'" fi - name: Run Enos scenario diff --git a/.github/workflows/godoc-test-checker.yml b/.github/workflows/godoc-test-checker.yml index 048042cf752a..c23f46b3fa5f 100644 --- a/.github/workflows/godoc-test-checker.yml +++ b/.github/workflows/godoc-test-checker.yml @@ -11,17 +11,13 @@ jobs: godoc-test-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c with: - ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - name: get metadata - id: get-metadata - run: echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT - name: Set Up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 with: cache: true - go-version: ${{ steps.get-metadata.outputs.go-version }} + go-version-file: ./.go-version - name: Verify new tests have go docs - run: make ci-vet-godoctests \ No newline at end of file + run: make ci-vet-godoctests diff --git a/.github/workflows/goversion-checker.yml b/.github/workflows/goversion-checker.yml deleted file mode 100644 index 71ed31b65e5f..000000000000 --- a/.github/workflows/goversion-checker.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Check Go version - -on: - pull_request: - types: [opened, synchronize] - -jobs: - go-version-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - name: Verify go versions in tree are consistent with one another - run: | - GOVER=$(cat .go-version) - EXPECTED="docker.mirror.hashicorp.services/cimg/go:$GOVER" - GOT=$(yq .references.environment.GO_IMAGE .circleci/config/executors/@executors.yml) - if [ "$EXPECTED" != "$GOT" ]; then - echo "version mismatch, .go-version has '$GOVER' and circleci config uses '$GOT'" - exit 1 - fi \ No newline at end of file diff --git a/.github/workflows/oss.yml b/.github/workflows/oss.yml index 4e03b9761ba4..d49550ff5a05 100644 --- a/.github/workflows/oss.yml +++ b/.github/workflows/oss.yml @@ -58,15 +58,15 @@ jobs: - 'ui/**' - name: "Default to core board" - run: echo "PROJECT=170" >> $GITHUB_ENV + run: echo "PROJECT=170" >> "$GITHUB_ENV" - if: github.event.pull_request != null && steps.changes.outputs.cryptosec == 'true' - run: echo "PROJECT=172" >> $GITHUB_ENV + run: echo "PROJECT=172" >> "$GITHUB_ENV" - if: github.event.pull_request != null && steps.changes.outputs.ecosystem == 'true' - run: echo "PROJECT=169" >> $GITHUB_ENV + run: echo "PROJECT=169" >> "$GITHUB_ENV" - if: github.event.pull_request != null && steps.changes.outputs.devex == 'true' - run: echo "PROJECT=176" >> $GITHUB_ENV + run: echo "PROJECT=176" >> "$GITHUB_ENV" - if: github.event.pull_request != null && steps.changes.outputs.ui == 'true' - run: echo "PROJECT=171" >> $GITHUB_ENV + run: echo "PROJECT=171" >> "$GITHUB_ENV" - uses: actions/add-to-project@v0.3.0 with: @@ -125,4 +125,4 @@ jobs: # ) { # deletedItemId # } - # }' -f project_id=$PROJECT_ID -f item_id=$item_id || true \ No newline at end of file + # }' -f project_id=$PROJECT_ID -f item_id=$item_id || true diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 5d61d8af333a..b00c84cfcfa9 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -10,8 +10,7 @@ on: jobs: scan: - runs-on: - labels: ['linux', 'large'] + runs-on: ['linux', 'large'] if: ${{ github.actor != 'dependabot[bot]' || github.actor != 'hc-github-team-secure-vault-core' }} steps: - uses: actions/checkout@v3 @@ -39,14 +38,14 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - mkdir $HOME/.bin - cd $GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep + mkdir "$HOME/.bin" + cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep" go build -o scan-plugin-semgrep . - mv scan-plugin-semgrep $HOME/.bin + mv scan-plugin-semgrep "$HOME/.bin" - cd $GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql + cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql" go build -o scan-plugin-codeql . - mv scan-plugin-codeql $HOME/.bin + mv scan-plugin-codeql "$HOME/.bin" # Semgrep python3 -m pip install semgrep @@ -54,11 +53,11 @@ jobs: # CodeQL LATEST=$(gh release list --repo https://github.com/github/codeql-action | cut -f 3 | sort --version-sort | tail -n1) gh release download --repo https://github.com/github/codeql-action --pattern codeql-bundle-linux64.tar.gz "$LATEST" - tar xf codeql-bundle-linux64.tar.gz -C $HOME/.bin + tar xf codeql-bundle-linux64.tar.gz -C "$HOME/.bin" # Add to PATH - echo "$HOME/.bin" >> $GITHUB_PATH - echo "$HOME/.bin/codeql" >> $GITHUB_PATH + echo "$HOME/.bin" >> "$GITHUB_PATH" + echo "$HOME/.bin/codeql" >> "$GITHUB_PATH" - name: Scan id: scan diff --git a/.github/workflows/setup-go-cache.yml b/.github/workflows/setup-go-cache.yml new file mode 100644 index 000000000000..3b8040a20545 --- /dev/null +++ b/.github/workflows/setup-go-cache.yml @@ -0,0 +1,33 @@ +on: + workflow_call: + inputs: + runs-on: + required: true + type: string +jobs: + setup-go-cache: + runs-on: ${{ fromJSON(inputs.runs-on) }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0 as of 2023-01-18 + - id: setup-go + name: Setup go + uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 #v3.4.0 as of 2022-12-07 + with: + go-version-file: ./.go-version + cache: true + - id: setup-git + name: Setup Git configuration + run: | + git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com".insteadOf https://github.com + - id: download-modules + name: Download go modules + run: | + # go list ./... forces downloading some additional versions of modules that 'go mod + # download' misses. We need this because we make use of go list itself during + # code generation in later builds that rely on this module cache. + go list ./... + go list -test ./... + + go mod download + ( cd sdk && go mod download ) + ( cd api && go mod download ) diff --git a/.github/workflows/test-ci-cleanup.yml b/.github/workflows/test-ci-cleanup.yml index 5035b86760c0..3afdb7229d23 100644 --- a/.github/workflows/test-ci-cleanup.yml +++ b/.github/workflows/test-ci-cleanup.yml @@ -22,7 +22,7 @@ jobs: - name: Get all regions id: setup run: | - echo "regions=$(aws ec2 describe-regions --region us-east-1 --output json --query 'Regions[].RegionName' | tr -d '\n ')" >> $GITHUB_OUTPUT + echo "regions=$(aws ec2 describe-regions --region us-east-1 --output json --query 'Regions[].RegionName' | tr -d '\n ')" >> "$GITHUB_OUTPUT" aws-nuke: needs: setup diff --git a/.github/workflows/test-enos-scenario-ui.yml b/.github/workflows/test-enos-scenario-ui.yml index b7d882cee137..9fa25bd0f3b2 100644 --- a/.github/workflows/test-enos-scenario-ui.yml +++ b/.github/workflows/test-enos-scenario-ui.yml @@ -42,16 +42,16 @@ jobs: env: IS_ENT: ${{ startsWith(github.event.repository.name, 'vault-enterprise' ) }} run: | - echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT - echo "node-version=$(cat ./ui/.nvmrc)" >> $GITHUB_OUTPUT - if ${IS_ENT} == true; then + echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT" + echo "node-version=$(cat ./ui/.nvmrc)" >> "$GITHUB_OUTPUT" + if [ "$IS_ENT" == true ]; then echo "detected vault_edition=ent" - echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> $GITHUB_OUTPUT - echo "vault_edition=ent" >> $GITHUB_OUTPUT + echo "runs-on=['self-hosted', 'ondemand', 'os=linux', 'type=m5d.4xlarge']" >> "$GITHUB_OUTPUT" + echo "vault_edition=ent" >> "$GITHUB_OUTPUT" else echo "detected vault_edition=oss" - echo "runs-on=\"custom-linux-xl-vault-latest\"" >> $GITHUB_OUTPUT - echo "vault_edition=oss" >> $GITHUB_OUTPUT + echo "runs-on=\"custom-linux-xl-vault-latest\"" >> "$GITHUB_OUTPUT" + echo "vault_edition=oss" >> "$GITHUB_OUTPUT" fi run-ui-tests: @@ -100,7 +100,7 @@ jobs: run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true - name: Check Chrome Installed id: chrome-check - run: echo "chrome-version=$(chrome --version 2> /dev/null || google-chrome --version 2> /dev/null || google-chrome-stable --version 2> /dev/null || echo 'not-installed')" >> $GITHUB_OUTPUT + run: echo "chrome-version=$(chrome --version 2> /dev/null || google-chrome --version 2> /dev/null || google-chrome-stable --version 2> /dev/null || echo 'not-installed')" >> "$GITHUB_OUTPUT" - name: Install Chrome Dependencies if: steps.chrome-check.outputs.chrome-version == 'not-installed' run: | diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml new file mode 100644 index 000000000000..ec12a8a2d061 --- /dev/null +++ b/.github/workflows/test-go.yml @@ -0,0 +1,180 @@ +on: + workflow_call: + inputs: + name: + description: A suffix to be added to the matrix job names. + required: true + type: string + go-arch: + description: The execution architecture (arm, amd64, etc.) + required: true + type: string + enterprise: + description: A flag indicating if this workflow is executing for the enterprise repository. + required: true + type: string + total-runners: + description: Number of runners to use for executing the tests on. + required: true + type: string + env-vars: + description: A map of environment variables as JSON. + required: false + type: string + default: '{}' + extra-flags: + description: A space-separated list of additional build flags. + required: false + type: string + runs-on: + description: An expression indicating which kind of runners to use. + required: false + type: string + default: ubuntu-latest + go-build-tags: + description: A comma-separated list of additional build tags to consider satisfied during the build. + required: false + type: string + +env: ${{ fromJSON(inputs.env-vars) }} + +jobs: + test-generate-test-package-list: + runs-on: ${{ fromJSON(inputs.runs-on) }} + name: Verify Test Package Distribution + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - id: test + working-directory: .github/scripts + run: | + ENTERPRISE=${{ inputs.enterprise }} ./test-generate-test-package-lists.sh + runner-indexes: + runs-on: ${{ fromJSON(inputs.runs-on) }} + name: Generate runner indexes + # + # This job generates a JSON Array of integers ranging from 1 to 16. + # That array is used in the matrix section of the test-go job below. + # + outputs: + runner-indexes: ${{ steps.generate-index-list.outputs.indexes }} + steps: + - id: generate-index-list + run: | + INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})" + INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")" + echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}" + test-go: + permissions: + id-token: write # Note: this permission is explicitly required for Vault auth + contents: read + name: "${{ matrix.runner-index }} ${{ inputs.name }}" + needs: + - runner-indexes + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + fail-fast: false + matrix: + # + # Initialize the runner-index key with the JSON array of integers + # generated above. + # + runner-index: ${{ fromJSON(needs.runner-indexes.outputs.runner-indexes) }} + env: + GOPRIVATE: github.com/hashicorp/* + TIMEOUT_IN_MINUTES: 60 + steps: + - id: setup-git + name: Setup Git configuration + run: | + git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com".insteadOf https://github.com + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 + with: + go-version-file: ./.go-version + cache: true + - name: Authenticate to Vault + id: vault-auth + if: github.repository == 'hashicorp/vault-enterprise' + run: vault-auth + - name: Fetch Secrets + id: secrets + if: github.repository == 'hashicorp/vault-enterprise' + uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI; + kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2; + kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS; + kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL; + kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID; + kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET; + kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID; + - id: run-go-tests + name: Run Go tests + timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }} + run: | + set -exo pipefail + + # + # This script creates a Bash array with 16 elements each + # containing a space delimited list of package names. The + # array element corresponding to this instance's + # matrix.runner-index value. + # + ENTERPRISE=${{ inputs.enterprise }} source .github/scripts/generate-test-package-lists.sh + + # Build the dynamically generated source files. + make prep + + mkdir -p test-results/go-test + + # We don't want VAULT_LICENSE set when running Go tests, because that's + # not what developers have in their environments and it could break some + # tests; it would be like setting VAULT_TOKEN. However some non-Go + # CI commands, like the UI tests, shouldn't have to worry about licensing. + # So we provide the tests which want an externally supplied license with licenses + # via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it. + # shellcheck disable=SC2034 + VAULT_LICENSE= + + # Assign test licenses to relevant variables if they aren't already + if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then + export VAULT_LICENSE_CI=${{ secrets.ci_license }} + export VAULT_LICENSE_2=${{ secrets.ci_license_2 }} + export HCP_API_ADDRESS=${{ secrets.HCP_API_ADDRESS }} + export HCP_AUTH_URL=${{ secrets.HCP_AUTH_URL }} + export HCP_CLIENT_ID=${{ secrets.HCP_CLIENT_ID }} + export HCP_CLIENT_SECRET=${{ secrets.HCP_CLIENT_SECRET }} + export HCP_RESOURCE_ID=${{ secrets.HCP_RESOURCE_ID }} + # Temporarily removing this variable to cause HCP Link tests + # to be skipped. + #export HCP_SCADA_ADDRESS=${{ secrets.HCP_SCADA_ADDRESS }} + fi + + # shellcheck disable=SC2086 # can't quote package list + GOARCH=${{ inputs.go-arch }} \ + go run gotest.tools/gotestsum --format=short-verbose \ + --junitfile test-results/go-test/results.xml \ + --jsonfile test-results/go-test/results.json \ + -- \ + -tags "${{ inputs.go-build-tags }}" \ + -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ + -parallel=20 \ + ${{ inputs.extra-flags }} \ + \ + ${test_packages[${{ matrix.runner-index }}]} + - name: Archive test results + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-results-${{ matrix.runner-index }} + path: test-results/ + if: always() + - name: Create a summary of tests + uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f + with: + paths: "test-results/go-test/results.xml" + show: "fail" + if: always() diff --git a/.github/workflows/test-run-acc-tests-for-path.yml b/.github/workflows/test-run-acc-tests-for-path.yml index c53fb1aa9537..f82b32c567a1 100644 --- a/.github/workflows/test-run-acc-tests-for-path.yml +++ b/.github/workflows/test-run-acc-tests-for-path.yml @@ -20,15 +20,13 @@ jobs: go-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - id: get-metadata - run: echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - name: Set Up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 with: - go-version: ${{ steps.get-metadata.outputs.go-version }} + go-version-file: ./.go-version - run: go test -v ./${{ inputs.path }}/... 2>&1 | tee ${{ inputs.name }}.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@b7f8abb1508181956e8e162db84b466c27e18ce with: name: ${{ inputs.name }}-output path: ${{ inputs.name }}.txt diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index a687f222af43..738f1e008189 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -77,10 +77,10 @@ jobs: ref: ${{ inputs.vault-revision }} - id: metadata run: | - echo "build-date=$(make ci-get-date)" >> $GITHUB_OUTPUT - echo "version=$(make ci-get-version)" >> $GITHUB_OUTPUT - filtered=$(make ci-filter-matrix) - echo "matrix=$(echo $filtered)}" >> $GITHUB_OUTPUT + # shellcheck disable=SC2129 + echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT" + echo "version=$(make ci-get-version)" >> "$GITHUB_OUTPUT" + echo "matrix=$(make ci-filter-matrix)" >> "$GITHUB_OUTPUT" # Run the Enos test scenarios run: diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 17309e55a9d7..f40519e53516 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -35,9 +35,7 @@ block() { # Add all check functions to this space separated list. # They are executed in this order (see end of file). -CHECKS="ui_lint circleci_verify" - -MIN_CIRCLECI_VERSION=0.1.5575 +CHECKS="ui_lint" # Run ui linter if changes in that dir detected. ui_lint() { @@ -62,82 +60,6 @@ ui_lint() { $LINTER || block "UI lint failed" } -# Check .circleci/config.yml is up to date and valid, and that all changes are -# included together in this commit. -circleci_verify() { - # Change to the root dir of the repo. - cd "$(git rev-parse --show-toplevel)" - - # Fail early if we accidentally used '.yaml' instead of '.yml' - if ! git diff --name-only --cached --exit-code -- '.circleci/***.yaml'; then - # This is just for consistency, as I keep making this mistake - Sam. - block "ERROR: File(s) with .yaml extension detected. Please rename them .yml instead." - fi - - # Succeed early if no changes to yml files in .circleci/ are currently staged. - # make ci-verify is slow so we really don't want to run it unnecessarily. - if git diff --name-only --cached --exit-code -- '.circleci/***.yml'; then - return 0 - fi - # Make sure to add no explicit output before this line, as it would just be noise - # for those making non-circleci changes. - echo "==> Verifying config changes in .circleci/" - echo "--> OK: All files are .yml not .yaml" - - # Ensure commit includes _all_ files in .circleci/ - # So not only are the files up to date, but we are also committing them in one go. - if ! git diff --name-only --exit-code -- '.circleci/***.yml'; then - echo "ERROR: Some .yml diffs in .circleci/ are staged, others not." - block "Please commit the entire .circleci/ directory together, or omit it altogether." - fi - - echo "--> OK: All .yml files in .circleci are staged." - - if ! REASON=$(check_circleci_cli_version); then - echo "*** WARNING: Unable to verify changes in .circleci/:" - echo "--> $REASON" - # We let this pass if there is no valid circleci version installed. - return 0 - fi - - if ! make -C .circleci ci-verify; then - block "ERROR: make ci-verify failed" - fi - - echo "--> OK: make ci-verify succeeded." -} - -check_circleci_cli_version() { - if ! command -v circleci > /dev/null 2>&1; then - echo "circleci cli not installed." - return 1 - fi - - CCI="circleci --skip-update-check" - - if ! THIS_VERSION=$($CCI version) > /dev/null 2>&1; then - # Guards against very old versions that do not have --skip-update-check. - echo "The installed circleci cli is too old. Please upgrade to at least $MIN_CIRCLECI_VERSION." - return 1 - fi - - # SORTED_MIN is the lower of the THIS_VERSION and MIN_CIRCLECI_VERSION. - if ! SORTED_MIN="$(printf "%s\n%s" "$MIN_CIRCLECI_VERSION" "$THIS_VERSION" | sort -V | head -n1)"; then - echo "Failed to sort versions. Please open an issue to report this." - return 1 - fi - - if [ "$THIS_VERSION" != "${THIS_VERSION#$MIN_CIRCLECI_VERSION}" ]; then - return 0 # OK - Versions have the same prefix, so we consider them equal. - elif [ "$SORTED_MIN" = "$MIN_CIRCLECI_VERSION" ]; then - return 0 # OK - MIN_CIRCLECI_VERSION is lower than THIS_VERSION. - fi - - # Version too low. - echo "The installed circleci cli v$THIS_VERSION is too old. Please upgrade to at least $MIN_CIRCLECI_VERSION" - return 1 -} - for CHECK in $CHECKS; do # Force each check into a subshell to avoid crosstalk. ( $CHECK ) || exit $? diff --git a/Makefile b/Makefile index ac68d55f0714..3cbd57751cce 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ fmtcheck: #@sh -c "'$(CURDIR)/scripts/gofmtcheck.sh'" fmt: - find . -name '*.go' | grep -v pb.go | grep -v vendor | xargs gofumpt -w + find . -name '*.go' | grep -v pb.go | grep -v vendor | xargs go run mvdan.cc/gofumpt -w semgrep: semgrep --include '*.go' --exclude 'vendor' -a -f tools/semgrep . @@ -257,13 +257,6 @@ hana-database-plugin: mongodb-database-plugin: @CGO_ENABLED=0 $(GO_CMD) build -o bin/mongodb-database-plugin ./plugins/database/mongodb/mongodb-database-plugin -.PHONY: ci-config -ci-config: - @$(MAKE) -C .circleci ci-config -.PHONY: ci-verify -ci-verify: - @$(MAKE) -C .circleci ci-verify - .PHONY: bin default prep test vet bootstrap ci-bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path packages build build-ci semgrep semgrep-ci vet-godoctests ci-vet-godoctests .NOTPARALLEL: ember-dist ember-dist-dev diff --git a/command/server/hcp_link_config_test.go b/command/server/hcp_link_config_test.go index f71c96d76cc5..51f5a5ec8800 100644 --- a/command/server/hcp_link_config_test.go +++ b/command/server/hcp_link_config_test.go @@ -9,6 +9,10 @@ import ( ) func TestHCPLinkConfig(t *testing.T) { + t.Setenv("HCP_CLIENT_ID", "") + t.Setenv("HCP_CLIENT_SECRET", "") + t.Setenv("HCP_RESOURCE_ID", "") + config, err := LoadConfigFile("./test-fixtures/hcp_link_config.hcl") if err != nil { t.Fatalf("err: %s", err) diff --git a/go.mod b/go.mod index 3ef3cf893bca..cc13601b7273 100644 --- a/go.mod +++ b/go.mod @@ -207,6 +207,7 @@ require ( google.golang.org/protobuf v1.28.1 gopkg.in/ory-am/dockertest.v3 v3.3.4 gopkg.in/square/go-jose.v2 v2.6.0 + gotest.tools/gotestsum v1.9.0 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed layeh.com/radius v0.0.0-20190322222518-890bc1058917 mvdan.cc/gofumpt v0.3.1 @@ -292,6 +293,7 @@ require ( github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect github.com/digitalocean/godo v1.7.5 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/cli v20.10.18+incompatible // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect @@ -301,6 +303,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gabriel-vasile/mimetype v1.3.1 // indirect github.com/gammazero/deque v0.0.0-20190130191400-2afb3858e9c7 // indirect github.com/gammazero/workerpool v0.0.0-20190406235159-88d534f22b56 // indirect diff --git a/go.sum b/go.sum index cee0f40a36fe..f3dbd5374b87 100644 --- a/go.sum +++ b/go.sum @@ -512,6 +512,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.8+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -589,8 +591,9 @@ github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.3.1 h1:qevA6c2MtE1RorlScnixeG0VA1H4xrXyhyX3oWBynNQ= github.com/gabriel-vasile/mimetype v1.3.1/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= @@ -1857,6 +1860,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw= @@ -2071,6 +2075,7 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -2102,6 +2107,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2215,6 +2221,7 @@ golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2235,6 +2242,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= @@ -2331,10 +2339,12 @@ golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2515,9 +2525,12 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A= +gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= +gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/serviceregistration/kubernetes/testing/testserver.go b/serviceregistration/kubernetes/testing/testserver.go index 50232a2e573e..4f406eb6871b 100644 --- a/serviceregistration/kubernetes/testing/testserver.go +++ b/serviceregistration/kubernetes/testing/testserver.go @@ -1,6 +1,7 @@ package testing import ( + _ "embed" "encoding/json" "fmt" "io/ioutil" @@ -18,15 +19,27 @@ import ( const ( ExpectedNamespace = "default" ExpectedPodName = "shell-demo" - - // File names of samples pulled from real life. - caCrtFile = "ca.crt" - respGetPod = "resp-get-pod.json" - respNotFound = "resp-not-found.json" - respUpdatePod = "resp-update-pod.json" - tokenFile = "token" ) +// Pull real-life-based testing data in from files at compile time. +// We decided to embed them in the test binary because of past issues +// with reading files that we encountered on CI workers. + +//go:embed ca.crt +var caCrt string + +//go:embed resp-get-pod.json +var getPodResponse string + +//go:embed resp-not-found.json +var notFoundResponse string + +//go:embed resp-update-pod.json +var updatePodTagsResponse string + +//go:embed token +var token string + var ( // ReturnGatewayTimeouts toggles whether the test server should return, // well, gateway timeouts... @@ -78,28 +91,6 @@ func Server(t *testing.T) (testState *State, testConf *Conf, closeFunc func()) { } } - // Read in our sample files. - token, err := readFile(tokenFile) - if err != nil { - t.Fatal(err) - } - caCrt, err := readFile(caCrtFile) - if err != nil { - t.Fatal(err) - } - notFoundResponse, err := readFile(respNotFound) - if err != nil { - t.Fatal(err) - } - getPodResponse, err := readFile(respGetPod) - if err != nil { - t.Fatal(err) - } - updatePodTagsResponse, err := readFile(respUpdatePod) - if err != nil { - t.Fatal(err) - } - // Plant our token in a place where it can be read for the config. tmpToken, err := ioutil.TempFile("", "token") if err != nil { diff --git a/tools/tools.go b/tools/tools.go index 9a4972b73c68..5f81033df61c 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -17,6 +17,7 @@ package tools //go:generate go install google.golang.org/grpc/cmd/protoc-gen-go-grpc //go:generate go install github.com/favadi/protoc-go-inject-tag //go:generate go install github.com/golangci/revgrep/cmd/revgrep +//go:generate go install gotest.tools/gotestsum import ( _ "golang.org/x/tools/cmd/goimports" @@ -31,4 +32,6 @@ import ( _ "github.com/favadi/protoc-go-inject-tag" _ "github.com/golangci/revgrep/cmd/revgrep" + + _ "gotest.tools/gotestsum" )