From 63c9fbf34171389d63f941b1c6057e48b6a6a4d6 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 15 Jan 2025 15:18:21 -0500 Subject: [PATCH] chore: update GitHub Actions workflows (#5109) Update workflows to sync up with ci-mgmt. --------- Co-authored-by: Pulumi Bot --- .github/actions/download-codegen/action.yml | 17 +++++ .github/actions/setup-tools/action.yml | 2 +- .github/workflows/build_sdk.yml | 4 +- .github/workflows/prerequisites.yml | 2 +- .github/workflows/test.yml | 5 -- .mk/minimal_schema.mk | 2 +- Makefile | 73 +++++++++++---------- 7 files changed, 61 insertions(+), 44 deletions(-) create mode 100644 .github/actions/download-codegen/action.yml diff --git a/.github/actions/download-codegen/action.yml b/.github/actions/download-codegen/action.yml new file mode 100644 index 00000000000..0cbde64bc1e --- /dev/null +++ b/.github/actions/download-codegen/action.yml @@ -0,0 +1,17 @@ +name: Download the code generator binary +description: Downloads the code generator binary to `bin/`. + +runs: + using: "composite" + steps: + + - name: Download the code generator binary for aws + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: pulumi-tfgen-aws + path: ${{ github.workspace }}/bin + + - name: Ensure the code generator binary for aws is executable + shell: bash + run: | + find ${{ github.workspace }} -name "pulumi-*-aws" -print -exec chmod +x {} \; diff --git a/.github/actions/setup-tools/action.yml b/.github/actions/setup-tools/action.yml index 49a7c5adebe..7a1d3f56d9e 100644 --- a/.github/actions/setup-tools/action.yml +++ b/.github/actions/setup-tools/action.yml @@ -47,7 +47,7 @@ runs: - name: Install Pulumi CLI if: inputs.tools == 'all' || contains(inputs.tools, 'pulumicli') - uses: pulumi/actions@c7fad9e2f0b79653172b36538b8b34b3c0291952 # v6 + uses: pulumi/actions@13b8b7177d6fb736766875dac9b78aab07bd785f # v6 with: pulumi-version: "dev" diff --git a/.github/workflows/build_sdk.yml b/.github/workflows/build_sdk.yml index e3e88108503..a510ec448ae 100644 --- a/.github/workflows/build_sdk.yml +++ b/.github/workflows/build_sdk.yml @@ -71,8 +71,8 @@ jobs: tools: pulumictl, pulumicli, ${{ matrix.language }} - name: Prepare local workspace run: make prepare_local_workspace - - name: Download tfgen - uses: ./.github/actions/download-tfgen + - name: Download codegen + uses: ./.github/actions/download-codegen - name: Update path run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - name: Restore makefile progress diff --git a/.github/workflows/prerequisites.yml b/.github/workflows/prerequisites.yml index d53da69dd78..61712ad6c1b 100644 --- a/.github/workflows/prerequisites.yml +++ b/.github/workflows/prerequisites.yml @@ -106,7 +106,7 @@ jobs: Maintainer note: consult the [runbook](https://github.com/pulumi/platform-providers-team/blob/main/playbooks/tf-provider-updating.md) for dealing with any breaking changes. - - name: Upload pulumi-tfgen-aws + - name: Upload codegen binary for aws uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: pulumi-tfgen-aws diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a087c291644..2bb40bd5807 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,11 +83,6 @@ jobs: pip3 install pipenv - name: Install dependencies run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.mk/minimal_schema.mk b/.mk/minimal_schema.mk index 91d5ab377cf..11023c60566 100644 --- a/.mk/minimal_schema.mk +++ b/.mk/minimal_schema.mk @@ -1,6 +1,6 @@ .PHONY: minimal_schema minimal_schema: - @(cd provider/cmd/pulumi-resource-aws && PULUMI_AWS_MINIMAL_SCHEMA=true VERSION=$(VERSION_GENERIC) go generate) + @(cd provider/cmd/pulumi-resource-aws && PULUMI_AWS_MINIMAL_SCHEMA=true GOOS='' GOARCH='' VERSION=$(VERSION_GENERIC) go generate) # In build_provider.yml workflow, minimal schema needs to be rebuilt right before the provider binary. bin/linux-amd64/$(PROVIDER): minimal_schema diff --git a/Makefile b/Makefile index b5541e27e41..41e4e55646a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ ORG := pulumi PROJECT := github.com/$(ORG)/pulumi-$(PACK) PROVIDER_PATH := provider/v6 VERSION_PATH := $(PROVIDER_PATH)/pkg/version.Version -TFGEN := pulumi-tfgen-$(PACK) +CODEGEN := pulumi-tfgen-$(PACK) PROVIDER := pulumi-resource-$(PACK) JAVA_GEN := pulumi-java-gen TESTPARALLELISM := 10 @@ -14,11 +14,17 @@ PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p 2 PULUMI_CONVERT := 1 PULUMI_MISSING_DOCS_ERROR := true +PULUMICTL_VERSION := v0.0.46 +PULUMICTL := $(shell which pulumictl || \ + (test ! -e $(WORKING_DIR)/bin/pulumictl && \ + GOPATH="$(WORKING_DIR)" go install "github.com/pulumi/pulumictl/cmd/pulumictl@$(PULUMICTL_VERSION)"; \ + echo "$(WORKING_DIR)/bin/puluimctl")) + # Override during CI using `make [TARGET] PROVIDER_VERSION=""` or by setting a PROVIDER_VERSION environment variable # Local & branch builds will just used this fixed default version unless specified PROVIDER_VERSION ?= 6.0.0-alpha.0+dev # Use this normalised version everywhere rather than the raw input to ensure consistency. -VERSION_GENERIC = $(shell pulumictl convert-version --language generic --version "$(PROVIDER_VERSION)") +VERSION_GENERIC = $(shell $(PULUMICTL) convert-version --language generic --version "$(PROVIDER_VERSION)") # Strips debug information from the provider binary to reduce its size and speed up builds LDFLAGS_STRIP_SYMBOLS=-s -w @@ -95,8 +101,8 @@ GEN_ENVS := PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(G generate_dotnet: .make/generate_dotnet build_dotnet: .make/build_dotnet .make/generate_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_dotnet: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/ +.make/generate_dotnet: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) dotnet --out sdk/dotnet/ cd sdk/dotnet/ && \ printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ echo "$(VERSION_GENERIC)" >version.txt @@ -109,8 +115,8 @@ build_dotnet: .make/build_dotnet generate_go: .make/generate_go build_go: .make/build_go .make/generate_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_go: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/ +.make/generate_go: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) go --out sdk/go/ @touch $@ .make/build_go: .make/generate_go cd sdk && go list "$$(grep -e "^module" go.mod | cut -d ' ' -f 2)/go/..." | xargs -I {} bash -c 'go build {} && go clean -i {}' @@ -136,8 +142,8 @@ build_java: .make/build_java generate_nodejs: .make/generate_nodejs build_nodejs: .make/build_nodejs .make/generate_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_nodejs: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/ +.make/generate_nodejs: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) nodejs --out sdk/nodejs/ printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/nodejs/go.mod @touch $@ .make/build_nodejs: .make/generate_nodejs @@ -151,8 +157,8 @@ build_nodejs: .make/build_nodejs generate_python: .make/generate_python build_python: .make/build_python .make/generate_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_python: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/ +.make/generate_python: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) python --out sdk/python/ printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/python/go.mod cp README.md sdk/python/ @touch $@ @@ -193,7 +199,7 @@ install_nodejs_sdk: .make/install_nodejs_sdk install_python_sdk: .PHONY: install_dotnet_sdk install_go_sdk install_java_sdk install_nodejs_sdk install_python_sdk -# Install Pulumi plugins required for TFGen to resolve references +# Install Pulumi plugins required for CODEGEN to resolve references install_plugins: .make/install_plugins .make/install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi .make/install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) @@ -222,12 +228,12 @@ lint_provider.fix: # `make provider_no_deps` builds the provider binary directly, without ensuring that # `cmd/pulumi-resource-aws/schema.json` is valid and up to date. # To create a release ready binary, you should use `make provider`. -build_provider_cmd = cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER) +build_provider_cmd = cd provider && CGO_ENABLED=0 go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o "$(1)" -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER) provider: bin/$(PROVIDER) provider_no_deps: - $(call build_provider_cmd) + $(call build_provider_cmd,$(WORKING_DIR)/bin/$(PROVIDER)) bin/$(PROVIDER): .make/schema - $(call build_provider_cmd) + $(call build_provider_cmd,$(WORKING_DIR)/bin/$(PROVIDER)) .PHONY: provider provider_no_deps test: export PATH := $(WORKING_DIR)/bin:$(PATH) @@ -253,13 +259,13 @@ tfgen_no_deps: .make/schema .make/schema: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache .make/schema: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT) .make/schema: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR) -.make/schema: bin/$(TFGEN) .make/install_plugins .make/upstream - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) +.make/schema: bin/$(CODEGEN) .make/install_plugins .make/upstream + $(WORKING_DIR)/bin/$(CODEGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION_GENERIC) go generate cmd/$(PROVIDER)/main.go) @touch $@ -tfgen_build_only: bin/$(TFGEN) -bin/$(TFGEN): provider/*.go provider/go.* .make/upstream - (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) +tfgen_build_only: bin/$(CODEGEN) +bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream + (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(CODEGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(CODEGEN)) .PHONY: tfgen schema tfgen_no_deps tfgen_build_only # Apply patches to the upstream submodule, if it exists @@ -277,7 +283,7 @@ endif .PHONY: upstream bin/pulumi-java-gen: .pulumi-java-gen.version - pulumictl download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java + $(PULUMICTL) download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java # To make an immediately observable change to .ci-mgmt.yaml: # @@ -309,7 +315,7 @@ ci-mgmt: .ci-mgmt.yaml # Start debug server for tfgen debug_tfgen: - dlv --listen=:2345 --headless=true --api-version=2 exec $(WORKING_DIR)/bin/$(TFGEN) -- schema --out provider/cmd/$(PROVIDER) + dlv --listen=:2345 --headless=true --api-version=2 exec $(WORKING_DIR)/bin/$(CODEGEN) -- schema --out provider/cmd/$(PROVIDER) .PHONY: debug_tfgen # Provider cross-platform build & packaging @@ -323,25 +329,24 @@ SKIP_SIGNING ?= # These targets assume that the schema-embed.json exists - it's generated by tfgen. # We disable CGO to ensure that the binary is statically linked. -bin/linux-amd64/$(PROVIDER): TARGET := linux-amd64 -bin/linux-arm64/$(PROVIDER): TARGET := linux-arm64 -bin/darwin-amd64/$(PROVIDER): TARGET := darwin-amd64 -bin/darwin-arm64/$(PROVIDER): TARGET := darwin-arm64 -bin/windows-amd64/$(PROVIDER).exe: TARGET := windows-amd64 +bin/linux-amd64/$(PROVIDER): export GOOS := linux +bin/linux-amd64/$(PROVIDER): export GOARCH := amd64 +bin/linux-arm64/$(PROVIDER): export GOOS := linux +bin/linux-arm64/$(PROVIDER): export GOARCH := arm64 +bin/darwin-amd64/$(PROVIDER): export GOOS := darwin +bin/darwin-amd64/$(PROVIDER): export GOARCH := amd64 +bin/darwin-arm64/$(PROVIDER): export GOOS := darwin +bin/darwin-arm64/$(PROVIDER): export GOARCH := arm64 +bin/windows-amd64/$(PROVIDER).exe: export GOOS := windows +bin/windows-amd64/$(PROVIDER).exe: export GOARCH := amd64 bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: bin/jsign-6.0.jar - @# check the TARGET is set - @test $(TARGET) - @cd provider && \ - export GOOS=$$(echo "$(TARGET)" | cut -d "-" -f 1) && \ - export GOARCH=$$(echo "$(TARGET)" | cut -d "-" -f 2) && \ - export CGO_ENABLED=0 && \ - go build -o "${WORKING_DIR}/$@" $(PULUMI_PROVIDER_BUILD_PARALLELISM) -ldflags "$(LDFLAGS)" "$(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)" + $(call build_provider_cmd,$(WORKING_DIR)/$@) @# Only sign windows binary if fully configured. @# Test variables set by joining with | between and looking for || showing at least one variable is empty. @# Move the binary to a temporary location and sign it there to avoid the target being up-to-date if signing fails. @set -e; \ - if [[ "${TARGET}" = "windows-amd64" && "${SKIP_SIGNING}" != "true" ]]; then \ + if [[ "${GOOS}-${GOARCH}" = "windows-amd64" && "${SKIP_SIGNING}" != "true" ]]; then \ if [[ "|${AZURE_SIGNING_CLIENT_ID}|${AZURE_SIGNING_CLIENT_SECRET}|${AZURE_SIGNING_TENANT_ID}|${AZURE_SIGNING_KEY_VAULT_URI}|" == *"||"* ]]; then \ echo "Can't sign windows binaries as required configuration not set: AZURE_SIGNING_CLIENT_ID, AZURE_SIGNING_CLIENT_SECRET, AZURE_SIGNING_TENANT_ID, AZURE_SIGNING_KEY_VAULT_URI"; \ echo "To rebuild with signing delete the unsigned $@ and rebuild with the fixed configuration"; \